diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardInitialScanStateProvider.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardInitialScanStateProvider.kt index 25a337b76968..06cd3809379d 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardInitialScanStateProvider.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardInitialScanStateProvider.kt @@ -22,6 +22,7 @@ import com.duckduckgo.di.scopes.ActivityScope import com.duckduckgo.pir.impl.dashboard.state.PirDashboardInitialScanStateProvider.DashboardBrokerWithStatus import com.duckduckgo.pir.impl.dashboard.state.PirDashboardInitialScanStateProvider.DashboardBrokerWithStatus.Status.COMPLETED import com.duckduckgo.pir.impl.dashboard.state.PirDashboardInitialScanStateProvider.DashboardBrokerWithStatus.Status.IN_PROGRESS +import com.duckduckgo.pir.impl.dashboard.state.PirDashboardInitialScanStateProvider.DashboardBrokerWithStatus.Status.NOT_STARTED import com.duckduckgo.pir.impl.store.PirRepository import com.duckduckgo.pir.impl.store.PirSchedulingRepository import com.squareup.anvil.annotations.ContributesBinding @@ -110,6 +111,6 @@ class RealPirDashboardInitialScanStateProvider @Inject constructor( } override suspend fun getScanResults(): List { - return getAllExtractedProfileResults() + return getAllExtractedProfileResults(includeResultsForDeprecatedProfileQueries = false) } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardMaintenanceScanDataProvider.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardMaintenanceScanDataProvider.kt index ee90873e6eda..2b140c31c19d 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardMaintenanceScanDataProvider.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardMaintenanceScanDataProvider.kt @@ -94,13 +94,13 @@ class RealPirDashboardMaintenanceScanDataProvider @Inject constructor( private val pirSchedulingRepository: PirSchedulingRepository, ) : PirDashboardStateProvider(currentTimeProvider, pirRepository, pirSchedulingRepository), PirDashboardMaintenanceScanDataProvider { override suspend fun getInProgressOptOuts(): List = withContext(dispatcherProvider.io()) { - return@withContext getAllExtractedProfileResults().filter { + return@withContext getAllExtractedProfileResults(includeResultsForDeprecatedProfileQueries = false).filter { it.optOutRemovedDateInMillis == null || it.optOutRemovedDateInMillis == 0L } } override suspend fun getRemovedOptOuts(): List = withContext(dispatcherProvider.io()) { - val allRemovedExtractedProfiles = getAllExtractedProfileResults().filter { + val allRemovedExtractedProfiles = getAllExtractedProfileResults(includeResultsForDeprecatedProfileQueries = true).filter { it.optOutRemovedDateInMillis != null && it.optOutRemovedDateInMillis != 0L } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardStateProvider.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardStateProvider.kt index 50e4bdf0fd31..72f2f485f53c 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardStateProvider.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/dashboard/state/PirDashboardStateProvider.kt @@ -33,9 +33,10 @@ abstract class PirDashboardStateProvider( private val pirRepository: PirRepository, private val pirSchedulingRepository: PirSchedulingRepository, ) { - suspend fun getAllExtractedProfileResults(): List { + suspend fun getAllExtractedProfileResults(includeResultsForDeprecatedProfileQueries: Boolean): List { + val nonDeprecatedProfileQueries = pirRepository.getValidUserProfileQueries().map { it.id }.toSet() val extractedProfiles = pirRepository.getAllExtractedProfiles().filter { - !it.deprecated + !it.deprecated && (includeResultsForDeprecatedProfileQueries || it.profileQueryId in nonDeprecatedProfileQueries) } val extractedProfilesFromBrokers = getAllExtractedProfileResultForBrokers(extractedProfiles) val extractedProfilesFromMirrorSites = extractedProfilesFromBrokers.getMirrorSites( diff --git a/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardInitialScanStateProviderTest.kt b/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardInitialScanStateProviderTest.kt index 0c4edd0e6d2a..1995713404b9 100644 --- a/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardInitialScanStateProviderTest.kt +++ b/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardInitialScanStateProviderTest.kt @@ -23,6 +23,7 @@ import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.Broker import com.duckduckgo.pir.impl.models.ExtractedProfile import com.duckduckgo.pir.impl.models.MirrorSite +import com.duckduckgo.pir.impl.models.ProfileQuery import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus import com.duckduckgo.pir.impl.models.scheduling.JobRecord.ScanJobRecord @@ -338,6 +339,7 @@ class RealPirDashboardInitialScanStateProviderTest { whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(emptyList()) // When val result = testee.getScanResults() @@ -396,11 +398,14 @@ class RealPirDashboardInitialScanStateProviderTest { ) val brokerOptOutUrls = mapOf("broker1" to "https://broker1.com/optout") + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(brokerOptOutUrls) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(optOutJobs) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) // When val result = testee.getScanResults() @@ -441,11 +446,14 @@ class RealPirDashboardInitialScanStateProviderTest { ), ) + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(optOutJobs) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) // When val result = testee.getScanResults() @@ -494,11 +502,14 @@ class RealPirDashboardInitialScanStateProviderTest { createBroker("broker3", parent = "broker1"), // broker2 is child of broker1 ) + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) // When val result = testee.getScanResults() @@ -532,11 +543,14 @@ class RealPirDashboardInitialScanStateProviderTest { ), ) + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(mirrorSites) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) // When val result = testee.getScanResults() @@ -553,6 +567,37 @@ class RealPirDashboardInitialScanStateProviderTest { assertEquals("https://mirror1.com/optout", mirrorResult.broker.optOutUrl) } + @Test + fun whenProfileQueryIsDeprecatedThenGetScanResultsFiltersItOut() = runTest { + // Given + val extractedProfiles = listOf( + createExtractedProfile(dbId = 1L, brokerName = "broker1", name = "John Doe", profileQueryId = 1L), + createExtractedProfile(dbId = 2L, brokerName = "broker1", name = "Jane Smith", profileQueryId = 2L), + createExtractedProfile(dbId = 3L, brokerName = "broker1", name = "Joe Smith", profileQueryId = 3L), + ) + val activeBrokers = listOf(createBroker("broker1")) + // Only profile queries 1 and 3 are valid (non-deprecated), profile query 2 is deprecated + val validProfileQueries = listOf( + createProfileQuery(id = 1L), + createProfileQuery(id = 3L), + ) + + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) + whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) + + // When + val result = testee.getScanResults() + + // Then - profile 2 (Jane Smith) should be filtered out because its profile query is deprecated + assertEquals(2, result.size) + assertEquals("John Doe", result[0].extractedProfile.name) + assertEquals("Joe Smith", result[1].extractedProfile.name) + } + private suspend fun setupForEmptyBrokersAndJobs() { whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) @@ -614,6 +659,7 @@ class RealPirDashboardInitialScanStateProviderTest { dbId: Long, brokerName: String, name: String, + profileQueryId: Long = 1L, age: String = "30", addresses: List = emptyList(), alternativeNames: List = emptyList(), @@ -623,7 +669,7 @@ class RealPirDashboardInitialScanStateProviderTest { ): ExtractedProfile { return ExtractedProfile( dbId = dbId, - profileQueryId = 1L, + profileQueryId = profileQueryId, brokerName = brokerName, name = name, alternativeNames = alternativeNames, @@ -652,4 +698,23 @@ class RealPirDashboardInitialScanStateProviderTest { removedAt = removedAt, ) } + + private fun createProfileQuery( + id: Long, + firstName: String = "John", + lastName: String = "Doe", + ): ProfileQuery { + return ProfileQuery( + id = id, + firstName = firstName, + lastName = lastName, + city = "New York", + state = "NY", + addresses = emptyList(), + birthYear = 1990, + fullName = "$firstName $lastName", + age = 34, + deprecated = false, + ) + } } diff --git a/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardMaintenanceScanDataProviderTest.kt b/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardMaintenanceScanDataProviderTest.kt index 11db611ee3e6..f193558a7f0a 100644 --- a/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardMaintenanceScanDataProviderTest.kt +++ b/pir/pir-impl/src/test/java/com/duckduckgo/pir/impl/dashboard/state/RealPirDashboardMaintenanceScanDataProviderTest.kt @@ -22,6 +22,7 @@ import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.Broker import com.duckduckgo.pir.impl.models.ExtractedProfile import com.duckduckgo.pir.impl.models.MirrorSite +import com.duckduckgo.pir.impl.models.ProfileQuery import com.duckduckgo.pir.impl.models.scheduling.BrokerSchedulingConfig import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus @@ -105,17 +106,20 @@ class RealPirDashboardMaintenanceScanDataProviderTest { ), ) + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(optOutJobs) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) // When val result = testee.getInProgressOptOuts() // Then - assertEquals(2, result.size) // Only profile 1 should be in progress + assertEquals(2, result.size) // Only profile 1 and 3 should be in progress (not removed) assertEquals("John Doe", result[0].extractedProfile.name) assertEquals(currentTime - 1000, result[0].optOutSubmittedDateInMillis) assertEquals(0L, result[0].optOutRemovedDateInMillis) @@ -124,6 +128,100 @@ class RealPirDashboardMaintenanceScanDataProviderTest { assertEquals(0L, result[0].optOutRemovedDateInMillis) } + @Test + fun whenProfileQueryIsDeprecatedThenGetInProgressOptOutsFiltersItOut() = runTest { + // Given + val extractedProfiles = listOf( + createExtractedProfile(dbId = 1L, brokerName = "broker1", name = "John Doe", profileQueryId = 1L), + createExtractedProfile(dbId = 2L, brokerName = "broker1", name = "Jane Smith", profileQueryId = 2L), + createExtractedProfile(dbId = 3L, brokerName = "broker1", name = "Joe Smith", profileQueryId = 3L), + ) + val activeBrokers = listOf(createBroker("broker1")) + val optOutJobs = listOf( + createOptOutJobRecord( + extractedProfileId = 1L, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = currentTime - 1000, + optOutRemovedDateInMillis = 0L, + ), + createOptOutJobRecord( + extractedProfileId = 2L, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = currentTime - 2000, + optOutRemovedDateInMillis = 0L, + ), + createOptOutJobRecord( + extractedProfileId = 3L, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = currentTime - 3000, + optOutRemovedDateInMillis = 0L, + ), + ) + // Only profile queries 1 and 3 are valid (non-deprecated), profile query 2 is deprecated + val validProfileQueries = listOf( + createProfileQuery(id = 1L), + createProfileQuery(id = 3L), + ) + + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) + whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(optOutJobs) + whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) + + // When + val result = testee.getInProgressOptOuts() + + // Then - profile 2 (Jane Smith) should be filtered out because its profile query is deprecated + assertEquals(2, result.size) + assertEquals("John Doe", result[0].extractedProfile.name) + assertEquals("Joe Smith", result[1].extractedProfile.name) + } + + @Test + fun whenProfileQueryIsDeprecatedThenGetRemovedOptOutsIncludesIt() = runTest { + // Given - getRemovedOptOuts should include results for deprecated profile queries + val extractedProfiles = listOf( + createExtractedProfile(dbId = 1L, brokerName = "broker1", name = "John Doe", profileQueryId = 1L), + createExtractedProfile(dbId = 2L, brokerName = "broker1", name = "Jane Smith", profileQueryId = 2L), + ) + val activeBrokers = listOf(createBroker("broker1")) + val optOutJobs = listOf( + createOptOutJobRecord( + extractedProfileId = 1L, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = currentTime - 2000, + optOutRemovedDateInMillis = currentTime - 500, + ), + createOptOutJobRecord( + extractedProfileId = 2L, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = currentTime - 3000, + optOutRemovedDateInMillis = currentTime - 1000, + ), + ) + // Only profile query 1 is valid (non-deprecated), profile query 2 is deprecated + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) + whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(optOutJobs) + whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) + + // When + val result = testee.getRemovedOptOuts() + + // Then - both should be included since getRemovedOptOuts includes deprecated profile queries + assertEquals(2, result.size) + val johnDoeResult = result.find { it.result.extractedProfile.name == "John Doe" }!! + assertEquals(currentTime - 500, johnDoeResult.result.optOutRemovedDateInMillis) + val janeSmithResult = result.find { it.result.extractedProfile.name == "Jane Smith" }!! + assertEquals(currentTime - 1000, janeSmithResult.result.optOutRemovedDateInMillis) + } + @Test fun whenNoRemovedOptOutsExistThenGetRemovedOptOutsReturnsEmptyList() = runTest { // Given @@ -180,11 +278,14 @@ class RealPirDashboardMaintenanceScanDataProviderTest { "broker2" to "https://broker2.com/optout", ) + val validProfileQueries = listOf(createProfileQuery(id = 1L)) + whenever(mockPirRepository.getAllExtractedProfiles()).thenReturn(extractedProfiles) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(activeBrokers) whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(brokerOptOutUrls) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(optOutJobs) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(validProfileQueries) // When val result = testee.getRemovedOptOuts() @@ -828,6 +929,7 @@ class RealPirDashboardMaintenanceScanDataProviderTest { whenever(mockPirRepository.getAllBrokerOptOutUrls()).thenReturn(emptyMap()) whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) whenever(mockPirRepository.getAllMirrorSites()).thenReturn(emptyList()) + whenever(mockPirRepository.getValidUserProfileQueries()).thenReturn(emptyList()) } private suspend fun setupForEmptyBrokersAndJobs() { @@ -891,6 +993,7 @@ class RealPirDashboardMaintenanceScanDataProviderTest { dbId: Long, brokerName: String, name: String, + profileQueryId: Long = 1L, age: String = "30", addresses: List = emptyList(), alternativeNames: List = emptyList(), @@ -900,7 +1003,7 @@ class RealPirDashboardMaintenanceScanDataProviderTest { ): ExtractedProfile { return ExtractedProfile( dbId = dbId, - profileQueryId = 1L, + profileQueryId = profileQueryId, brokerName = brokerName, name = name, alternativeNames = alternativeNames, @@ -945,4 +1048,23 @@ class RealPirDashboardMaintenanceScanDataProviderTest { maxAttempts = maxAttempts, ) } + + private fun createProfileQuery( + id: Long, + firstName: String = "John", + lastName: String = "Doe", + ): ProfileQuery { + return ProfileQuery( + id = id, + firstName = firstName, + lastName = lastName, + city = "New York", + state = "NY", + addresses = emptyList(), + birthYear = 1990, + fullName = "$firstName $lastName", + age = 34, + deprecated = false, + ) + } }