From 84d8ee22bf3ba6c0d8e5bac5911a200007397d41 Mon Sep 17 00:00:00 2001 From: Steffen Ohrendorf Date: Wed, 15 Oct 2025 19:09:12 +0200 Subject: [PATCH 1/5] drop missing entities in case of stale lucene data Signed-off-by: Steffen Ohrendorf --- .../resources/v1/SearchResource.java | 3 +- .../FuzzyVulnerableSoftwareSearchManager.java | 26 ++++--- ...zyVulnerableSoftwareSearchManagerTest.java | 78 +++++++++++++++++-- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/dependencytrack/resources/v1/SearchResource.java b/src/main/java/org/dependencytrack/resources/v1/SearchResource.java index 0b7cf4954..43bfb31bb 100644 --- a/src/main/java/org/dependencytrack/resources/v1/SearchResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/SearchResource.java @@ -202,8 +202,7 @@ public class SearchResource extends AlpineResource { @PermissionRequired(Permissions.Constants.VIEW_PORTFOLIO) public Response vulnerableSoftwareSearch(@QueryParam("query") String query, @QueryParam("cpe") String cpe) { if (StringUtils.isNotBlank(cpe)) { - final FuzzyVulnerableSoftwareSearchManager searchManager = new FuzzyVulnerableSoftwareSearchManager(false); - final SearchResult searchResult = searchManager.searchIndex(FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(cpe)); + final SearchResult searchResult = FuzzyVulnerableSoftwareSearchManager.searchIndex(FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(cpe)); return Response.ok(searchResult).build(); } else { final SearchResult searchResult = SearchManager.searchVulnerableSoftwareIndex(query, 1000); diff --git a/src/main/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManager.java b/src/main/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManager.java index d48dd2d3c..615bcc5c1 100644 --- a/src/main/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManager.java +++ b/src/main/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManager.java @@ -49,12 +49,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import static org.dependencytrack.search.IndexConstants.VULNERABLESOFTWARE_UUID; + public class FuzzyVulnerableSoftwareSearchManager { private static final Logger LOGGER = Logger.getLogger(FuzzyVulnerableSoftwareSearchManager.class); @@ -147,7 +148,8 @@ public class FuzzyVulnerableSoftwareSearchManager { } return fuzzyList; } - private List fuzzySearch(QueryManager qm, Part part, String vendor, String product) { + + private static List fuzzySearch(QueryManager qm, Part part, String vendor, String product) { try { String sanitizedVendor = vendor.replace(" ", "_"); String sanitizedProduct = product.replace(" ", "_"); @@ -160,7 +162,7 @@ public class FuzzyVulnerableSoftwareSearchManager { } } - public SearchResult searchIndex(final String luceneQuery) { + public static SearchResult searchIndex(final String luceneQuery) { final SearchResult searchResult = new SearchResult(); final List> resultSet = new ArrayList<>(); IndexManager indexManager = VulnerableSoftwareIndexer.getInstance(); @@ -218,15 +220,17 @@ public class FuzzyVulnerableSoftwareSearchManager { return searchResult; } - private List fuzzySearch(QueryManager qm, String luceneQuery) { - List fuzzyList = new LinkedList<>(); - SearchResult sr = searchIndex(luceneQuery); - if (sr.getResults().containsKey("vulnerablesoftware")) { - for (Map result : sr.getResults().get("vulnerablesoftware")) { - fuzzyList.add(qm.getObjectByUuid(VulnerableSoftware.class, result.get("uuid"))); - } + static List fuzzySearch(QueryManager qm, String luceneQuery) { + final var vulnerableSoftware = searchIndex(luceneQuery).getResults().get("vulnerablesoftware"); + if (vulnerableSoftware == null) { + return Collections.emptyList(); } - return fuzzyList; + + return vulnerableSoftware + .stream() + .map(result -> qm.getObjectByUuid(VulnerableSoftware.class, result.get(VULNERABLESOFTWARE_UUID))) + .filter(Objects::nonNull) + .toList(); } public static String getLuceneCpeRegexp(String cpeString) { diff --git a/src/test/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManagerTest.java b/src/test/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManagerTest.java index cec05004c..3127d51e1 100644 --- a/src/test/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManagerTest.java +++ b/src/test/java/org/dependencytrack/search/FuzzyVulnerableSoftwareSearchManagerTest.java @@ -22,12 +22,19 @@ import us.springett.parsers.cpe.values.Part; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.regex.Pattern; +import static org.dependencytrack.search.IndexConstants.VULNERABLESOFTWARE_UUID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class FuzzyVulnerableSoftwareSearchManagerTest { @@ -81,10 +88,10 @@ class FuzzyVulnerableSoftwareSearchManagerTest { us.springett.parsers.cpe.Cpe justThePart = new us.springett.parsers.cpe.Cpe(Part.APPLICATION, "*", "*", "*", "*", "*", "*", "*", "*", "*", "*"); // wildcard all components after part to constrain fuzzing to components of same type e.g. application, operating-system String fuzzyTerm = FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(justThePart.toCpe23FS()); - SearchResult searchResult = toTest.searchIndex("product:libexpat1~0.88 AND " + fuzzyTerm); + SearchResult searchResult = FuzzyVulnerableSoftwareSearchManager.searchIndex("product:libexpat1~0.88 AND " + fuzzyTerm); // Oddly validating lucene first cuz can't decouple from that. - Assertions.assertEquals(1, searchResult.getResults().size()); - Assertions.assertEquals(1, searchResult.getResults().values().iterator().next().size()); + assertEquals(1, searchResult.getResults().size()); + assertEquals(1, searchResult.getResults().values().iterator().next().size()); Component component = new Component(); component.setName("libexpat1"); @@ -92,7 +99,7 @@ class FuzzyVulnerableSoftwareSearchManagerTest { Cpe cpe = CpeParser.parse(component.getCpe()); List vs = toTest.fuzzyAnalysis(qm, component, cpe); Assertions.assertFalse(vs.isEmpty()); - Assertions.assertSame(VALUE_TO_MATCH, vs.get(0)); + assertSame(VALUE_TO_MATCH, vs.get(0)); } @@ -100,9 +107,9 @@ class FuzzyVulnerableSoftwareSearchManagerTest { void getLuceneCpeRegexp() throws CpeValidationException, CpeEncodingException { us.springett.parsers.cpe.Cpe os = new us.springett.parsers.cpe.Cpe( Part.OPERATING_SYSTEM, "vendor", "product", "1\\.0", "2", "33","en", "inside", "Vista", "x86", "other"); - Assertions.assertEquals("cpe23:/cpe\\:2\\.3\\:a\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp("cpe:2.3:a:*:*:*:*:*:*:*:*:*:*")); - Assertions.assertEquals("cpe23:/cpe\\:2\\.3\\:o\\:vendor\\:product\\:1.0\\:2\\:33\\:en\\:inside\\:vista\\:x86\\:other/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe23FS())); - Assertions.assertEquals("cpe22:/cpe\\:\\/o\\:vendor\\:product\\:1.0\\:2\\:33\\:en/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe22Uri())); + assertEquals("cpe23:/cpe\\:2\\.3\\:a\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp("cpe:2.3:a:*:*:*:*:*:*:*:*:*:*")); + assertEquals("cpe23:/cpe\\:2\\.3\\:o\\:vendor\\:product\\:1.0\\:2\\:33\\:en\\:inside\\:vista\\:x86\\:other/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe23FS())); + assertEquals("cpe22:/cpe\\:\\/o\\:vendor\\:product\\:1.0\\:2\\:33\\:en/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe22Uri())); } @Test @@ -117,6 +124,63 @@ class FuzzyVulnerableSoftwareSearchManagerTest { "cpe:2.3:a:*:file:*:*:*:*:*:file:*:*").matches()); } + @Test + void fuzzySearchDropsMissingEntities() { + var qm = mock(QueryManager.class); + + var id1 = UUID.randomUUID(); + var id2 = UUID.randomUUID(); + var results = List.of( + Map.of(VULNERABLESOFTWARE_UUID, id1.toString()), + Map.of(VULNERABLESOFTWARE_UUID, id2.toString()) + ); + + var vs = mock(VulnerableSoftware.class); + when(qm.getObjectByUuid(VulnerableSoftware.class, id1.toString())).thenReturn(null); + when(qm.getObjectByUuid(VulnerableSoftware.class, id2.toString())).thenReturn(vs); + + var searchResult = mock(SearchResult.class); + when(searchResult.getResults()).thenReturn(Map.of("vulnerablesoftware", results)); + + List fuzzyResult; + try (var fvssm = mockStatic(FuzzyVulnerableSoftwareSearchManager.class)) { + fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query")).thenReturn(searchResult); + fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query")).thenCallRealMethod(); + + fuzzyResult = FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query"); + + fvssm.verify(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query")); + } + + verify(qm).getObjectByUuid(VulnerableSoftware.class, id1.toString()); + verify(qm).getObjectByUuid(VulnerableSoftware.class, id2.toString()); + + assertEquals(1, fuzzyResult.size()); + assertSame(vs, fuzzyResult.getFirst()); + } + + @Test + void fuzzySearchReturnsEmptyListIfNoResults() { + var qm = mock(QueryManager.class); + + var searchResult = mock(SearchResult.class); + when(searchResult.getResults()).thenReturn(Map.of()); + + List fuzzyResult; + try (var fvssm = mockStatic(FuzzyVulnerableSoftwareSearchManager.class)) { + fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query")).thenReturn(searchResult); + fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query")).thenCallRealMethod(); + + fuzzyResult = FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query"); + + fvssm.verify(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query")); + } + + verify(qm, never()).getObjectByUuid(any(), anyString()); + + assertEquals(0, fuzzyResult.size()); + } + private static void commitIndex() { IndexManagerTestUtil.commitIndex(VulnerableSoftwareIndexer.getInstance()); } From 5c7cad6b6b277dae32450435a6401df12590712c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 08:01:23 +0000 Subject: [PATCH 2/5] build(deps): bump com.google.cloud.sql:postgres-socket-factory Bumps com.google.cloud.sql:postgres-socket-factory from 1.25.3 to 1.26.1. --- updated-dependencies: - dependency-name: com.google.cloud.sql:postgres-socket-factory dependency-version: 1.26.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eaa0ce850..6df50de86 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ 12.0.1 1.25.3 1.25.3 - 1.25.3 + 1.26.1 3.0.0 1.28.0 1.14.0 From e7fcc354ab19b7e1414704200a4fa56a4164c793 Mon Sep 17 00:00:00 2001 From: Steffen Ohrendorf Date: Fri, 17 Oct 2025 18:51:50 +0200 Subject: [PATCH 3/5] improve vulnerablesoftware cpe normalization performance Signed-off-by: Steffen Ohrendorf --- .../org/dependencytrack/upgrade/v4135/v4135Updater.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java b/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java index 0cc90b681..a76026bea 100644 --- a/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java +++ b/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java @@ -43,7 +43,13 @@ public class v4135Updater extends AbstractUpgradeItem { try (final Statement statement = connection.createStatement()) { LOGGER.info("Normalizing \"VULNERABLESOFTWARE\" CPE columns"); statement.execute(/* language=SQL */ """ - UPDATE "VULNERABLESOFTWARE" SET "PART" = LOWER("PART"), "VENDOR" = LOWER("VENDOR"), "PRODUCT" = LOWER("PRODUCT") + UPDATE "VULNERABLESOFTWARE" + SET "PART" = LOWER("PART"), + "VENDOR" = LOWER("VENDOR"), + "PRODUCT" = LOWER("PRODUCT") + WHERE "PART <> LOWER("PART") + OR "VENDOR <> LOWER("VENDOR") + OR "PRODUCT <> LOWER("PRODUCT")" """); } } From 0634ca393e0b2e5372145f9a2a857a031bbe425d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 17:24:44 +0000 Subject: [PATCH 4/5] build(deps): bump com.google.cloud.sql:mysql-socket-factory-connector-j-8 Bumps com.google.cloud.sql:mysql-socket-factory-connector-j-8 from 1.25.3 to 1.26.1. --- updated-dependencies: - dependency-name: com.google.cloud.sql:mysql-socket-factory-connector-j-8 dependency-version: 1.26.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6df50de86..82aef7db4 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,7 @@ 0.1.2 12.0.1 1.25.3 - 1.25.3 + 1.26.1 1.26.1 3.0.0 1.28.0 From e7b117d7244cea1e0598933ff85eb871983e5741 Mon Sep 17 00:00:00 2001 From: Steffen Ohrendorf Date: Fri, 17 Oct 2025 19:34:20 +0200 Subject: [PATCH 5/5] remove sneaky double quote Signed-off-by: Steffen Ohrendorf --- .../org/dependencytrack/upgrade/v4135/v4135Updater.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java b/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java index a76026bea..286ed57f9 100644 --- a/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java +++ b/src/main/java/org/dependencytrack/upgrade/v4135/v4135Updater.java @@ -47,9 +47,9 @@ public class v4135Updater extends AbstractUpgradeItem { SET "PART" = LOWER("PART"), "VENDOR" = LOWER("VENDOR"), "PRODUCT" = LOWER("PRODUCT") - WHERE "PART <> LOWER("PART") - OR "VENDOR <> LOWER("VENDOR") - OR "PRODUCT <> LOWER("PRODUCT")" + WHERE "PART" <> LOWER("PART") + OR "VENDOR" <> LOWER("VENDOR") + OR "PRODUCT" <> LOWER("PRODUCT") """); } }