Compare commits

..

10 commits

Author SHA1 Message Date
Niklas
2688b1f90f
Merge pull request #5420 from stohrendorf/fix-cpe-fix 2025-10-17 20:44:57 +02:00
Steffen Ohrendorf
e7b117d724
remove sneaky double quote
Signed-off-by: Steffen Ohrendorf <steffen.ohrendorf@gmx.de>
2025-10-17 19:50:22 +02:00
Niklas
6d2b08a233
Merge pull request #5415 from DependencyTrack/dependabot/maven/com.google.cloud.sql-mysql-socket-factory-connector-j-8-1.26.1 2025-10-17 19:39:34 +02:00
dependabot[bot]
0634ca393e
build(deps): bump com.google.cloud.sql:mysql-socket-factory-connector-j-8
Bumps com.google.cloud.sql:mysql-socket-factory-connector-j-8 from 1.25.3 to 1.26.1.

---
updated-dependencies:
- dependency-name: com.google.cloud.sql:mysql-socket-factory-connector-j-8
  dependency-version: 1.26.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-10-17 17:24:44 +00:00
Niklas
389377f46b
Merge pull request #5418 from stohrendorf/speed-up-vulnerablesoftware-normalization 2025-10-17 19:24:04 +02:00
Niklas
60400bab82
Merge pull request #5416 from DependencyTrack/dependabot/maven/com.google.cloud.sql-postgres-socket-factory-1.26.1 2025-10-17 19:22:46 +02:00
Niklas
febf77bce2
Merge pull request #5408 from stohrendorf/stale-lucene-data 2025-10-17 19:22:23 +02:00
Steffen Ohrendorf
e7fcc354ab
improve vulnerablesoftware cpe normalization performance
Signed-off-by: Steffen Ohrendorf <steffen.ohrendorf@gmx.de>
2025-10-17 19:10:35 +02:00
dependabot[bot]
5c7cad6b6b
build(deps): bump com.google.cloud.sql:postgres-socket-factory
Bumps com.google.cloud.sql:postgres-socket-factory from 1.25.3 to 1.26.1.

---
updated-dependencies:
- dependency-name: com.google.cloud.sql:postgres-socket-factory
  dependency-version: 1.26.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-10-17 08:01:23 +00:00
Steffen Ohrendorf
84d8ee22bf
drop missing entities in case of stale lucene data
Signed-off-by: Steffen Ohrendorf <steffen.ohrendorf@gmx.de>
2025-10-15 19:09:12 +02:00
5 changed files with 96 additions and 23 deletions

View file

@ -91,8 +91,8 @@
<lib.brotli-decoder.version>0.1.2</lib.brotli-decoder.version>
<lib.checkstyle.version>12.0.1</lib.checkstyle.version>
<lib.cloud-sql-connector-jdbc-sqlserver.version>1.25.3</lib.cloud-sql-connector-jdbc-sqlserver.version>
<lib.cloud-sql-mysql-socket-factory-connector-j-8.version>1.25.3</lib.cloud-sql-mysql-socket-factory-connector-j-8.version>
<lib.cloud-sql-postgres-socket-factory.version>1.25.3</lib.cloud-sql-postgres-socket-factory.version>
<lib.cloud-sql-mysql-socket-factory-connector-j-8.version>1.26.1</lib.cloud-sql-mysql-socket-factory-connector-j-8.version>
<lib.cloud-sql-postgres-socket-factory.version>1.26.1</lib.cloud-sql-postgres-socket-factory.version>
<lib.cpe-parser.version>3.0.0</lib.cpe-parser.version>
<lib.commons-compress.version>1.28.0</lib.commons-compress.version>
<lib.commons-text.version>1.14.0</lib.commons-text.version>

View file

@ -202,8 +202,7 @@ public class SearchResource extends AlpineResource {
@PermissionRequired(Permissions.Constants.VIEW_PORTFOLIO)
public Response vulnerableSoftwareSearch(@QueryParam("query") String query, @QueryParam("cpe") String cpe) {
if (StringUtils.isNotBlank(cpe)) {
final FuzzyVulnerableSoftwareSearchManager searchManager = new FuzzyVulnerableSoftwareSearchManager(false);
final SearchResult searchResult = searchManager.searchIndex(FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(cpe));
final SearchResult searchResult = FuzzyVulnerableSoftwareSearchManager.searchIndex(FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(cpe));
return Response.ok(searchResult).build();
} else {
final SearchResult searchResult = SearchManager.searchVulnerableSoftwareIndex(query, 1000);

View file

@ -49,12 +49,13 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static org.dependencytrack.search.IndexConstants.VULNERABLESOFTWARE_UUID;
public class FuzzyVulnerableSoftwareSearchManager {
private static final Logger LOGGER = Logger.getLogger(FuzzyVulnerableSoftwareSearchManager.class);
@ -147,7 +148,8 @@ public class FuzzyVulnerableSoftwareSearchManager {
}
return fuzzyList;
}
private List<VulnerableSoftware> fuzzySearch(QueryManager qm, Part part, String vendor, String product) {
private static List<VulnerableSoftware> fuzzySearch(QueryManager qm, Part part, String vendor, String product) {
try {
String sanitizedVendor = vendor.replace(" ", "_");
String sanitizedProduct = product.replace(" ", "_");
@ -160,7 +162,7 @@ public class FuzzyVulnerableSoftwareSearchManager {
}
}
public SearchResult searchIndex(final String luceneQuery) {
public static SearchResult searchIndex(final String luceneQuery) {
final SearchResult searchResult = new SearchResult();
final List<Map<String, String>> resultSet = new ArrayList<>();
IndexManager indexManager = VulnerableSoftwareIndexer.getInstance();
@ -218,15 +220,17 @@ public class FuzzyVulnerableSoftwareSearchManager {
return searchResult;
}
private List<VulnerableSoftware> fuzzySearch(QueryManager qm, String luceneQuery) {
List<VulnerableSoftware> fuzzyList = new LinkedList<>();
SearchResult sr = searchIndex(luceneQuery);
if (sr.getResults().containsKey("vulnerablesoftware")) {
for (Map<String, String> result : sr.getResults().get("vulnerablesoftware")) {
fuzzyList.add(qm.getObjectByUuid(VulnerableSoftware.class, result.get("uuid")));
}
static List<VulnerableSoftware> fuzzySearch(QueryManager qm, String luceneQuery) {
final var vulnerableSoftware = searchIndex(luceneQuery).getResults().get("vulnerablesoftware");
if (vulnerableSoftware == null) {
return Collections.emptyList();
}
return fuzzyList;
return vulnerableSoftware
.stream()
.map(result -> qm.getObjectByUuid(VulnerableSoftware.class, result.get(VULNERABLESOFTWARE_UUID)))
.filter(Objects::nonNull)
.toList();
}
public static String getLuceneCpeRegexp(String cpeString) {

View file

@ -43,7 +43,13 @@ public class v4135Updater extends AbstractUpgradeItem {
try (final Statement statement = connection.createStatement()) {
LOGGER.info("Normalizing \"VULNERABLESOFTWARE\" CPE columns");
statement.execute(/* language=SQL */ """
UPDATE "VULNERABLESOFTWARE" SET "PART" = LOWER("PART"), "VENDOR" = LOWER("VENDOR"), "PRODUCT" = LOWER("PRODUCT")
UPDATE "VULNERABLESOFTWARE"
SET "PART" = LOWER("PART"),
"VENDOR" = LOWER("VENDOR"),
"PRODUCT" = LOWER("PRODUCT")
WHERE "PART" <> LOWER("PART")
OR "VENDOR" <> LOWER("VENDOR")
OR "PRODUCT" <> LOWER("PRODUCT")
""");
}
}

View file

@ -22,12 +22,19 @@ import us.springett.parsers.cpe.values.Part;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.regex.Pattern;
import static org.dependencytrack.search.IndexConstants.VULNERABLESOFTWARE_UUID;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class FuzzyVulnerableSoftwareSearchManagerTest {
@ -81,10 +88,10 @@ class FuzzyVulnerableSoftwareSearchManagerTest {
us.springett.parsers.cpe.Cpe justThePart = new us.springett.parsers.cpe.Cpe(Part.APPLICATION, "*", "*", "*", "*", "*", "*", "*", "*", "*", "*");
// wildcard all components after part to constrain fuzzing to components of same type e.g. application, operating-system
String fuzzyTerm = FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(justThePart.toCpe23FS());
SearchResult searchResult = toTest.searchIndex("product:libexpat1~0.88 AND " + fuzzyTerm);
SearchResult searchResult = FuzzyVulnerableSoftwareSearchManager.searchIndex("product:libexpat1~0.88 AND " + fuzzyTerm);
// Oddly validating lucene first cuz can't decouple from that.
Assertions.assertEquals(1, searchResult.getResults().size());
Assertions.assertEquals(1, searchResult.getResults().values().iterator().next().size());
assertEquals(1, searchResult.getResults().size());
assertEquals(1, searchResult.getResults().values().iterator().next().size());
Component component = new Component();
component.setName("libexpat1");
@ -92,7 +99,7 @@ class FuzzyVulnerableSoftwareSearchManagerTest {
Cpe cpe = CpeParser.parse(component.getCpe());
List<VulnerableSoftware> vs = toTest.fuzzyAnalysis(qm, component, cpe);
Assertions.assertFalse(vs.isEmpty());
Assertions.assertSame(VALUE_TO_MATCH, vs.get(0));
assertSame(VALUE_TO_MATCH, vs.get(0));
}
@ -100,9 +107,9 @@ class FuzzyVulnerableSoftwareSearchManagerTest {
void getLuceneCpeRegexp() throws CpeValidationException, CpeEncodingException {
us.springett.parsers.cpe.Cpe os = new us.springett.parsers.cpe.Cpe( Part.OPERATING_SYSTEM, "vendor", "product", "1\\.0", "2", "33","en", "inside", "Vista", "x86", "other");
Assertions.assertEquals("cpe23:/cpe\\:2\\.3\\:a\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp("cpe:2.3:a:*:*:*:*:*:*:*:*:*:*"));
Assertions.assertEquals("cpe23:/cpe\\:2\\.3\\:o\\:vendor\\:product\\:1.0\\:2\\:33\\:en\\:inside\\:vista\\:x86\\:other/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe23FS()));
Assertions.assertEquals("cpe22:/cpe\\:\\/o\\:vendor\\:product\\:1.0\\:2\\:33\\:en/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe22Uri()));
assertEquals("cpe23:/cpe\\:2\\.3\\:a\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*\\:.*/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp("cpe:2.3:a:*:*:*:*:*:*:*:*:*:*"));
assertEquals("cpe23:/cpe\\:2\\.3\\:o\\:vendor\\:product\\:1.0\\:2\\:33\\:en\\:inside\\:vista\\:x86\\:other/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe23FS()));
assertEquals("cpe22:/cpe\\:\\/o\\:vendor\\:product\\:1.0\\:2\\:33\\:en/", FuzzyVulnerableSoftwareSearchManager.getLuceneCpeRegexp(os.toCpe22Uri()));
}
@Test
@ -117,6 +124,63 @@ class FuzzyVulnerableSoftwareSearchManagerTest {
"cpe:2.3:a:*:file:*:*:*:*:*:file:*:*").matches());
}
@Test
void fuzzySearchDropsMissingEntities() {
var qm = mock(QueryManager.class);
var id1 = UUID.randomUUID();
var id2 = UUID.randomUUID();
var results = List.of(
Map.of(VULNERABLESOFTWARE_UUID, id1.toString()),
Map.of(VULNERABLESOFTWARE_UUID, id2.toString())
);
var vs = mock(VulnerableSoftware.class);
when(qm.getObjectByUuid(VulnerableSoftware.class, id1.toString())).thenReturn(null);
when(qm.getObjectByUuid(VulnerableSoftware.class, id2.toString())).thenReturn(vs);
var searchResult = mock(SearchResult.class);
when(searchResult.getResults()).thenReturn(Map.of("vulnerablesoftware", results));
List<VulnerableSoftware> fuzzyResult;
try (var fvssm = mockStatic(FuzzyVulnerableSoftwareSearchManager.class)) {
fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query")).thenReturn(searchResult);
fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query")).thenCallRealMethod();
fuzzyResult = FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query");
fvssm.verify(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query"));
}
verify(qm).getObjectByUuid(VulnerableSoftware.class, id1.toString());
verify(qm).getObjectByUuid(VulnerableSoftware.class, id2.toString());
assertEquals(1, fuzzyResult.size());
assertSame(vs, fuzzyResult.getFirst());
}
@Test
void fuzzySearchReturnsEmptyListIfNoResults() {
var qm = mock(QueryManager.class);
var searchResult = mock(SearchResult.class);
when(searchResult.getResults()).thenReturn(Map.of());
List<VulnerableSoftware> fuzzyResult;
try (var fvssm = mockStatic(FuzzyVulnerableSoftwareSearchManager.class)) {
fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query")).thenReturn(searchResult);
fvssm.when(() -> FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query")).thenCallRealMethod();
fuzzyResult = FuzzyVulnerableSoftwareSearchManager.fuzzySearch(qm, "query");
fvssm.verify(() -> FuzzyVulnerableSoftwareSearchManager.searchIndex("query"));
}
verify(qm, never()).getObjectByUuid(any(), anyString());
assertEquals(0, fuzzyResult.size());
}
private static void commitIndex() {
IndexManagerTestUtil.commitIndex(VulnerableSoftwareIndexer.getInstance());
}