Skip to content

🐛 Do not use SHA lookup and use artifact list instead #852

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed

🐛 Do not use SHA lookup and use artifact list instead #852

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6ed93ac
Do not use SHA lookup and use artifact list instead
jmle Jun 26, 2025
c73b06a
Improvements
jmle Jun 26, 2025
0ed68f7
Improvement in algorithm
jmle Jun 27, 2025
8c5f4a5
Move archive related code back to util.go
jmle Jun 30, 2025
bd5d014
Fix toDependency
jmle Jun 30, 2025
15f4892
Fix setDepLabels
jmle Jul 1, 2025
34ba203
Remove dep for pom creation if not complete
jmle Jul 2, 2025
04e83b0
Try demo-output...
jmle Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 46 additions & 62 deletions demo-output.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,14 +219,6 @@
data: dependency
innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
- uri: file:///examples/java-project/pom.xml
message: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
codeSnip: "11 <url>http://www.konveyor.io</url>\n12 \n13 <properties>\n14 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15 </properties>\n16 \n17 <dependencies>\n18 \n19 <dependency>\n20 <groupId>io.javaoperatorsdk.operator</groupId>\n21 <artifactId>sample</artifactId>\n22 <version>0.0.0</version>\n23 </dependency>\n24 \n25 </dependencies>\n26 \n27 <build>\n28 </build>\n29 </project>\n"
lineNumber: 20
variables:
data: dependency
innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n "
matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
- uri: file:///examples/java-project/quarkus-1-6-2-jar-exploded/META-INF/maven/io.javaoperatorsdk/quarkus/pom.xml
message: <groupId>io.javaoperatorsdk</groupId><artifactId>operator-framework-quarkus-extension</artifactId><version>${project.version}</version>
codeSnip: "30 <version>${quarkus.version}</version>\n31 <type>pom</type>\n32 <scope>import</scope>\n33 </dependency>\n34 </dependencies>\n35 </dependencyManagement>\n36 \n37 <dependencies>\n38 <dependency>\n39 <groupId>io.javaoperatorsdk</groupId>\n40 <artifactId>operator-framework-quarkus-extension</artifactId>\n41 <version>${project.version}</version>\n42 </dependency>\n43 <dependency>\n44 <groupId>io.javaoperatorsdk</groupId>\n45 <artifactId>operator-framework-samples-common</artifactId>\n46 <version>${project.version}</version>\n47 </dependency>\n48 </dependencies>\n49 \n50 <build>"
Expand Down Expand Up @@ -623,6 +615,52 @@
name: javax.activation.activation
version: "1.1"
effort: 1
node-sample-rule-001:
description: Testing that the node provider works - type
category: potential
incidents:
- uri: file:///examples/nodejs/test_a.ts
message: nodejs sample rule 001
codeSnip: " 1 export interface Greeter {\n 2 name: string;\n 3 hello(): string;\n 4 }\n 5 \n 6 export const greeter: Greeter = {\n 7 name: \"Person1\",\n 8 hello() {\n 9 return `Hello, I'm ${this.name}`;\n10 },\n11 };\n"
lineNumber: 5
variables:
file: file:///examples/nodejs/test_a.ts
- uri: file:///examples/nodejs/test_b.ts
message: nodejs sample rule 001
codeSnip: " 1 import { greeter } from './test_a';\n 2 \n 3 console.log(greeter.hello());\n"
lineNumber: 0
variables:
file: file:///examples/nodejs/test_b.ts
- uri: file:///examples/nodejs/test_b.ts
message: nodejs sample rule 001
codeSnip: " 1 import { greeter } from './test_a';\n 2 \n 3 console.log(greeter.hello());\n"
lineNumber: 2
variables:
file: file:///examples/nodejs/test_b.ts
effort: 1
node-sample-rule-002:
description: Testing that the node provider works - function
category: potential
incidents:
- uri: file:///examples/nodejs/test_a.ts
message: nodejs sample rule 002
codeSnip: " 1 export interface Greeter {\n 2 name: string;\n 3 hello(): string;\n 4 }\n 5 \n 6 export const greeter: Greeter = {\n 7 name: \"Person1\",\n 8 hello() {\n 9 return `Hello, I'm ${this.name}`;\n10 },\n11 };\n"
lineNumber: 2
variables:
file: file:///examples/nodejs/test_a.ts
- uri: file:///examples/nodejs/test_a.ts
message: nodejs sample rule 002
codeSnip: " 1 export interface Greeter {\n 2 name: string;\n 3 hello(): string;\n 4 }\n 5 \n 6 export const greeter: Greeter = {\n 7 name: \"Person1\",\n 8 hello() {\n 9 return `Hello, I'm ${this.name}`;\n10 },\n11 };\n"
lineNumber: 7
variables:
file: file:///examples/nodejs/test_a.ts
- uri: file:///examples/nodejs/test_b.ts
message: nodejs sample rule 002
codeSnip: " 1 import { greeter } from './test_a';\n 2 \n 3 console.log(greeter.hello());\n"
lineNumber: 2
variables:
file: file:///examples/nodejs/test_b.ts
effort: 1
python-sample-rule-001:
description: ""
category: potential
Expand Down Expand Up @@ -831,14 +869,6 @@
data: dependency
innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
- uri: file:///examples/java-project/pom.xml
message: POM XML dependencies - '<groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>'
codeSnip: "11 <url>http://www.konveyor.io</url>\n12 \n13 <properties>\n14 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15 </properties>\n16 \n17 <dependencies>\n18 \n19 <dependency>\n20 <groupId>io.javaoperatorsdk.operator</groupId>\n21 <artifactId>sample</artifactId>\n22 <version>0.0.0</version>\n23 </dependency>\n24 \n25 </dependencies>\n26 \n27 <build>\n28 </build>\n29 </project>\n"
lineNumber: 20
variables:
data: dependency
innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n "
matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
- uri: file:///examples/java-project/quarkus-1-6-2-jar-exploded/META-INF/maven/io.javaoperatorsdk/quarkus/pom.xml
message: POM XML dependencies - '<groupId>io.javaoperatorsdk</groupId><artifactId>operator-framework-quarkus-extension</artifactId><version>${project.version}</version>'
codeSnip: "30 <version>${quarkus.version}</version>\n31 <type>pom</type>\n32 <scope>import</scope>\n33 </dependency>\n34 </dependencies>\n35 </dependencyManagement>\n36 \n37 <dependencies>\n38 <dependency>\n39 <groupId>io.javaoperatorsdk</groupId>\n40 <artifactId>operator-framework-quarkus-extension</artifactId>\n41 <version>${project.version}</version>\n42 </dependency>\n43 <dependency>\n44 <groupId>io.javaoperatorsdk</groupId>\n45 <artifactId>operator-framework-samples-common</artifactId>\n46 <version>${project.version}</version>\n47 </dependency>\n48 </dependencies>\n49 \n50 <build>"
Expand Down Expand Up @@ -980,52 +1010,6 @@
matchingXML: ""
effort: 1
node-sample-rule-001:
description: Testing that the node provider works - type
category: potential
incidents:
- uri: file:///examples/nodejs/test_a.ts
message: nodejs sample rule 001
codeSnip: " 1 export interface Greeter {\n 2 name: string;\n 3 hello(): string;\n 4 }\n 5 \n 6 export const greeter: Greeter = {\n 7 name: \"Person1\",\n 8 hello() {\n 9 return `Hello, I'm ${this.name}`;\n10 },\n11 };\n"
lineNumber: 5
variables:
file: file:///examples/nodejs/test_a.ts
- uri: file:///examples/nodejs/test_b.ts
message: nodejs sample rule 001
codeSnip: " 1 import { greeter } from './test_a';\n 2 \n 3 console.log(greeter.hello());\n"
lineNumber: 0
variables:
file: file:///examples/nodejs/test_b.ts
- uri: file:///examples/nodejs/test_b.ts
message: nodejs sample rule 001
codeSnip: " 1 import { greeter } from './test_a';\n 2 \n 3 console.log(greeter.hello());\n"
lineNumber: 2
variables:
file: file:///examples/nodejs/test_b.ts
effort: 1
node-sample-rule-002:
description: Testing that the node provider works - function
category: potential
incidents:
- uri: file:///examples/nodejs/test_a.ts
message: nodejs sample rule 002
codeSnip: " 1 export interface Greeter {\n 2 name: string;\n 3 hello(): string;\n 4 }\n 5 \n 6 export const greeter: Greeter = {\n 7 name: \"Person1\",\n 8 hello() {\n 9 return `Hello, I'm ${this.name}`;\n10 },\n11 };\n"
lineNumber: 2
variables:
file: file:///examples/nodejs/test_a.ts
- uri: file:///examples/nodejs/test_a.ts
message: nodejs sample rule 002
codeSnip: " 1 export interface Greeter {\n 2 name: string;\n 3 hello(): string;\n 4 }\n 5 \n 6 export const greeter: Greeter = {\n 7 name: \"Person1\",\n 8 hello() {\n 9 return `Hello, I'm ${this.name}`;\n10 },\n11 };\n"
lineNumber: 7
variables:
file: file:///examples/nodejs/test_a.ts
- uri: file:///examples/nodejs/test_b.ts
message: nodejs sample rule 002
codeSnip: " 1 import { greeter } from './test_a';\n 2 \n 3 console.log(greeter.hello());\n"
lineNumber: 2
variables:
file: file:///examples/nodejs/test_b.ts
effort: 1
insights:
field-rule-00001:
description: Sample field declaration rule
Expand Down
86 changes: 39 additions & 47 deletions external-providers/java-external-provider/pkg/java_external_provider/dependency.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"runtime"
"strings"

"github.com/go-logr/logr"
"github.com/konveyor/analyzer-lsp/engine/labels"
"github.com/konveyor/analyzer-lsp/output/v1/konveyor"
"github.com/konveyor/analyzer-lsp/provider"
Expand Down Expand Up @@ -437,8 +438,6 @@ func (p *javaServiceClient) getDependenciesForGradle(_ context.Context) (map[uri
lines := strings.Split(string(output), "\n")
deps := p.parseGradleDependencyOutput(lines)

// TODO: do we need to separate by submodule somehow?

path := p.findGradleBuild()
file := uri.File(path)
m := map[uri.URI][]provider.DepDAGItem{}
Expand Down Expand Up @@ -617,6 +616,7 @@ func (p *javaServiceClient) discoverDepsFromJars(path string, ll map[uri.URI][]k
m2RepoPath: p.mvnLocalRepo,
seen: map[string]bool{},
initialPath: path,
log: p.log,
}
filepath.WalkDir(path, w.walkDirForJar)
}
Expand All @@ -628,6 +628,7 @@ type walker struct {
initialPath string
seen map[string]bool
pomPaths []string
log logr.Logger
}

func (w *walker) walkDirForJar(path string, info fs.DirEntry, err error) error {
Expand All @@ -646,7 +647,7 @@ func (w *walker) walkDirForJar(path string, info fs.DirEntry, err error) error {
d := provider.Dep{
Name: info.Name(),
}
artifact, _ := toDependency(context.TODO(), path)
artifact, _ := toDependency(context.TODO(), w.log, w.depToLabels, path)
if (artifact != javaArtifact{}) {
d.Name = fmt.Sprintf("%s.%s", artifact.GroupId, artifact.ArtifactId)
d.Version = artifact.Version
Expand Down Expand Up @@ -708,6 +709,7 @@ func (p *javaServiceClient) discoverPoms(pathStart string, ll map[uri.URI][]konv
seen: map[string]bool{},
initialPath: pathStart,
pomPaths: []string{},
log: p.log,
}
filepath.WalkDir(pathStart, w.walkDirForPom)
return w.pomPaths
Expand Down Expand Up @@ -880,98 +882,88 @@ func (p *javaServiceClient) parseMavenDepLines(lines []string, localRepoPath, po
return []provider.DepDAGItem{}, nil
}

// depInit loads a map of package patterns and their associated labels for easy lookup
func (p *javaServiceClient) depInit() error {
err := p.initOpenSourceDepLabels()
if err != nil {
p.log.V(5).Error(err, "failed to initialize dep labels lookup for open source packages")
return err
}

err = p.initExcludeDepLabels()
if err != nil {
p.log.V(5).Error(err, "failed to initialize dep labels lookup for excluded packages")
return err
}

return nil
}

// initOpenSourceDepLabels reads user provided file that has a list of open source
// packages (supports regex) and loads a map of patterns -> labels for easy lookup
func (p *javaServiceClient) initOpenSourceDepLabels() error {
func initOpenSourceDepLabels(log logr.Logger, providerSpecificConfig map[string]interface{}) (map[string]*depLabelItem, error) {
var ok bool
var v interface{}
if v, ok = p.config.ProviderSpecificConfig[providerSpecificConfigOpenSourceDepListKey]; !ok {
p.log.V(7).Info("Did not find open source dep list.")
return nil
if v, ok = providerSpecificConfig[providerSpecificConfigOpenSourceDepListKey]; !ok {
log.V(7).Info("Did not find open source dep list.")
return nil, nil
}

var filePath string
if filePath, ok = v.(string); !ok {
return fmt.Errorf("unable to determine filePath from open source dep list")
return nil, fmt.Errorf("unable to determine filePath from open source dep list")
}

fileInfo, err := os.Stat(filePath)
if err != nil {
//TODO(shawn-hurley): consider wrapping error with value
return err
return nil, err
}

if fileInfo.IsDir() {
return fmt.Errorf("open source dep list must be a file, not a directory")
return nil, fmt.Errorf("open source dep list must be a file, not a directory")
}

file, err := os.Open(filePath)
if err != nil {
return err
return nil, err
}
defer file.Close()
return loadDepLabelItems(file, p.depToLabels,
labels.AsString(provider.DepSourceLabel, javaDepSourceOpenSource))
items, err := loadDepLabelItems(file, labels.AsString(provider.DepSourceLabel, javaDepSourceOpenSource), nil)
return items, nil
}

// initExcludeDepLabels reads user provided list of excluded packages
// and initiates label lookup for them
func (p *javaServiceClient) initExcludeDepLabels() error {
func initExcludeDepLabels(log logr.Logger, providerSpecificConfig map[string]interface{}, depToLabels map[string]*depLabelItem) (map[string]*depLabelItem, error) {
var ok bool
var v interface{}
if v, ok = p.config.ProviderSpecificConfig[providerSpecificConfigExcludePackagesKey]; !ok {
p.log.V(7).Info("did not find exclude packages list")
return nil
if v, ok = providerSpecificConfig[providerSpecificConfigExcludePackagesKey]; !ok {
log.V(7).Info("did not find exclude packages list")
return depToLabels, nil
}
var excludePackages []string
if excludePackages, ok = v.([]string); !ok {
return fmt.Errorf("%s config must be a list of packages to exclude", providerSpecificConfigExcludePackagesKey)
excludePackages, ok := v.([]string)
if !ok {
return nil, fmt.Errorf("%s config must be a list of packages to exclude", providerSpecificConfigExcludePackagesKey)
}
return loadDepLabelItems(strings.NewReader(
strings.Join(excludePackages, "\n")), p.depToLabels, provider.DepExcludeLabel)
items, err := loadDepLabelItems(strings.NewReader(strings.Join(excludePackages, "\n")), provider.DepExcludeLabel, depToLabels)
if err != nil {
return nil, err
}
return items, nil
}

// loadDepLabelItems reads list of patterns from reader and appends given
// label to the list of labels for the associated pattern
func loadDepLabelItems(r io.Reader, depToLabels map[string]*depLabelItem, label string) error {
func loadDepLabelItems(r io.Reader, label string, depToLabels map[string]*depLabelItem) (map[string]*depLabelItem, error) {
depToLabelsItems := map[string]*depLabelItem{}
if depToLabels != nil {
depToLabelsItems = depToLabels
}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
pattern := scanner.Text()
r, err := regexp.Compile(pattern)
if err != nil {
return fmt.Errorf("unable to create regexp for string: %v", pattern)
return nil, fmt.Errorf("unable to create regexp for string: %v", pattern)
}
//Make sure that we are not adding duplicates
if _, found := depToLabels[pattern]; !found {
depToLabels[pattern] = &depLabelItem{
if _, found := depToLabelsItems[pattern]; !found {
depToLabelsItems[pattern] = &depLabelItem{
r: r,
labels: map[string]interface{}{
label: nil,
},
}
} else {
if depToLabels[pattern].labels == nil {
depToLabels[pattern].labels = map[string]interface{}{}
if depToLabelsItems[pattern].labels == nil {
depToLabelsItems[pattern].labels = map[string]interface{}{}
}
depToLabels[pattern].labels[label] = nil
depToLabelsItems[pattern].labels[label] = nil
}
}
return nil
return depToLabelsItems, nil
}
Loading
Loading