Skip to content

Linkage monitor to read file to get list of artifacts for a repository (when necessary) #1961

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
suztomo wants to merge 3 commits into from
Closed

Linkage monitor to read file to get list of artifacts for a repository (when necessary) #1961

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
17aeb2d
linkage-monitor-artifacts.txt to tell list of artifacts
suztomo Feb 26, 2021
1f7fb70
javadoc
suztomo Feb 26, 2021
6a349b4
format
suztomo Feb 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 66 additions & 17 deletions ...itor/src/main/java/com/google/cloud/tools/dependencies/linkagemonitor/LinkageMonitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import com.google.common.collect.Sets;
import com.google.common.io.MoreFiles;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
Expand Down Expand Up @@ -82,15 +83,23 @@
public class LinkageMonitor {
private static final Logger logger = Logger.getLogger(LinkageMonitor.class.getName());

/**
* The file that Linkage Monitor reads to get list of the artifacts that the repository produces.
*/
private static final String LOCAL_ARTIFACT_FILE_NAME = "linkage-monitor-artifacts.txt";

private static final DefaultModelBuilder modelBuilder =
new DefaultModelBuilderFactory().newInstance();

// Finding latest version requires metadata from remote repository
private final RepositorySystem repositorySystem = RepositoryUtility.newRepositorySystem();
private final RepositorySystemSession session = RepositoryUtility.newSession(repositorySystem);
private static final RepositorySystem repositorySystem = RepositoryUtility.newRepositorySystem();
private static final RepositorySystemSession session =
RepositoryUtility.newSession(repositorySystem);
private final ImmutableMap<String, String> localArtifacts =
findLocalArtifacts(repositorySystem, session, Paths.get(".").toAbsolutePath());

private LinkageMonitor() throws IOException, MavenRepositoryException {}

public static void main(String[] arguments)
throws RepositoryException, IOException, MavenRepositoryException, ModelBuildingException {

Expand Down Expand Up @@ -146,12 +155,13 @@ private static void printUsageAndDie() {

/**
* Returns a map from versionless coordinates to the versions of the Maven coordinates in all
* pom.xml and the {@code dependencyManagement} section of BOMs found in {@code projectDirectory}.
* pom.xml. The map also contains the artifact list in the {@code linkage-monitor-artifacts.txt}
* if the file exists in {@code projectDirectory}.
*/
@VisibleForTesting
static ImmutableMap<String, String> findLocalArtifacts(
RepositorySystem repositorySystem, RepositorySystemSession session, Path projectDirectory) {
// The same coordinates may be added multiple times by an artifact and an entry in a BOM.
RepositorySystem repositorySystem, RepositorySystemSession session, Path projectDirectory)
throws IOException, MavenRepositoryException {
Map<String, String> artifactToVersion = new HashMap<>();
Iterable<Path> paths = MoreFiles.fileTraverser().breadthFirst(projectDirectory);

Expand Down Expand Up @@ -193,27 +203,66 @@ static ImmutableMap<String, String> findLocalArtifacts(
String versionlessCoordinates = model.getGroupId() + ":" + model.getArtifactId();
artifactToVersion.put(versionlessCoordinates, model.getVersion());
logger.fine("Found local artifact: " + model);
DependencyManagement dependencyManagement = model.getDependencyManagement();
if ("pom".equals(model.getPackaging()) && dependencyManagement != null) {
// Read the content of a BOM.
for (org.apache.maven.model.Dependency dependency :
dependencyManagement.getDependencies()) {
String managedDependencyVersionlessCoordinates =
dependency.getGroupId() + ":" + dependency.getArtifactId();
artifactToVersion.put(managedDependencyVersionlessCoordinates, dependency.getVersion());
logger.fine("Found local artifact in the BOM: " + dependency);
}
}

} catch (ModelBuildingException ex) {
// Maven may fail to build pom.xml files found in irrelevant directories, such as "target"
// and "test" directories of the project. Such failures can be ignored.
logger.info("Ignoring bad model: " + path + ": " + ex.getMessage());
}
}

// For gax-java (Gradle) repository, which does not have pom.xml, linkage-monitor-artifacts.txt
// tells which artifacts to use for the snapshot BOM.
ImmutableMap<String, String> localArtifactsFromFile =
findLocalArtifactsFromFile(projectDirectory.resolve(LOCAL_ARTIFACT_FILE_NAME));
artifactToVersion.putAll(localArtifactsFromFile);

return ImmutableMap.copyOf(artifactToVersion);
}

/**
* Returns a map from versionless coordinates to the versions of the Maven coordinates by checking
* the latest version for the versionless coordinates listed in {@code file}. If the file does not
* exists, it returns an empty map.
*
* @throws IOException if the file contains line in an invalid format for versionless coordinates
* or there's no version found for the coordinates.
*/
@VisibleForTesting
static ImmutableMap<String, String> findLocalArtifactsFromFile(Path file)
throws IOException, MavenRepositoryException {
if (!Files.exists(file)) {
return ImmutableMap.of();
}

List<String> lines = Files.readAllLines(file);

ImmutableMap.Builder<String, String> artifactToVersion = ImmutableMap.builder();
for (String line : lines) {
String[] elements = line.split(":");
if (elements.length != 2) {
throw new IOException(
"Invalid format in "
+ LOCAL_ARTIFACT_FILE_NAME
+ ". The file should contain only versionless coordinates.");
}
String groupId = elements[0];
String artifactId = elements[1];
// The latest version comes at last
ImmutableList<String> versions =
RepositoryUtility.findVersions(repositorySystem, groupId, artifactId);
if (versions.isEmpty()) {
// Not using ArtifactNotFoundException, because an artifact requires a version
throw new IOException("Could not find any version for " + line);
}
String latestVersion = versions.get(versions.size() - 1);
logger.fine("Found artifact from file: " + line + ":" + latestVersion);

artifactToVersion.put(line, latestVersion);
}

return artifactToVersion.build();
}

/**
* Returns new problems in the BOM specified by {@code groupId} and {@code artifactId}. This
* method compares the latest release of the BOM and its snapshot version which uses artifacts in
Expand Down
69 changes: 63 additions & 6 deletions .../src/test/java/com/google/cloud/tools/dependencies/linkagemonitor/LinkageMonitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,25 +272,29 @@ public void testBuildModelWithSnapshotBom_JdkVersionActivation()
}

@Test
public void testFindLocalArtifacts() {
public void testFindLocalArtifacts() throws IOException, MavenRepositoryException {
ImmutableMap<String, String> localArtifacts =
LinkageMonitor.findLocalArtifacts(
system, session, Paths.get("src/test/resources/testproject"));

// This should not include project under "build" directory, but should include the entries
// in the dependencyManagement section of the gax-bom/pom.xml.
Truth.assertThat(localArtifacts).hasSize(6);
Truth.assertThat(localArtifacts)
.doesNotContainKey("com.google.cloud.tools:copy-of-test-subproject-in-build");
assertEquals("0.0.1-SNAPSHOT", localArtifacts.get("com.google.cloud.tools:test-project"));
assertEquals("0.0.2-SNAPSHOT", localArtifacts.get("com.google.cloud.tools:test-subproject"));
assertEquals("1.60.2-SNAPSHOT", localArtifacts.get("com.google.api:gax-bom"));
assertEquals(
"localArtifacts should contain the dependency management section in the BOM",
"1.60.2-SNAPSHOT",
localArtifacts.get("com.google.api:gax"));

// Linkage Monitor should read linkage-monitor-artifacts.txt. The file tells versionless
// coordinates
Truth.assertThat(localArtifacts).containsKey("com.google.api:gax");
Truth.assertThat(localArtifacts).containsKey("com.google.api:gax-grpc");
Truth.assertThat(localArtifacts).containsKey("com.google.api:gax-httpjson");
}

@Test
public void testFindLocalArtifacts_absolutePath() {
public void testFindLocalArtifacts_absolutePath() throws IOException, MavenRepositoryException {
Path relativePath = Paths.get("src/test/resources/testproject");
Path absolutePath = relativePath.toAbsolutePath();
ImmutableMap<String, String> localArtifactsFromAbsolutePath =
Expand All @@ -304,4 +308,57 @@ public void testFindLocalArtifacts_absolutePath() {
localArtifactsFromRelativePath,
localArtifactsFromAbsolutePath);
}

@Test
public void testFindLocalArtifactsFromFile() throws IOException, MavenRepositoryException {
Path artifactFile = Paths.get("src/test/resources/testproject/linkage-monitor-artifacts.txt");
ImmutableMap<String, String> localArtifacts =
LinkageMonitor.findLocalArtifactsFromFile(artifactFile);

Truth.assertThat(localArtifacts.keySet())
.containsExactly(
"com.google.api:gax", "com.google.api:gax-grpc", "com.google.api:gax-httpjson");
}

@Test
public void testFindLocalArtifactsFromFile_invalidLines()
throws IOException, MavenRepositoryException {
// This file has coordinates with versions
Path artifactFile =
Paths.get("src/test/resources/testproject/linkage-monitor-artifacts-invalid-format.txt");

try {
LinkageMonitor.findLocalArtifactsFromFile(artifactFile);
fail();
} catch (IOException ex) {
// pass
}
}

@Test
public void testFindLocalArtifactsFromFile_nonexistentCoordinates()
throws IOException, MavenRepositoryException {
// There's no such artifact with "com.google.api:gax-nonexistent-coordinates"
Path artifactFile =
Paths.get(
"src/test/resources/testproject/linkage-monitor-artifacts-nonexistent-coordinates.txt");

try {
LinkageMonitor.findLocalArtifactsFromFile(artifactFile);
fail();
} catch (IOException ex) {
// pass
}
}

@Test
public void testFindLocalArtifactsFromFile_nonexistentFile()
throws IOException, MavenRepositoryException {
// Most of the repositories do not have the file. Linkage Monitor should not throw an exception.
Path artifactFile = Paths.get("src/test/resources/testproject/nonexistent-file");

ImmutableMap<String, String> localArtifactsFromFile =
LinkageMonitor.findLocalArtifactsFromFile(artifactFile);
Truth.assertThat(localArtifactsFromFile).isEmpty();
}
}
28 changes: 0 additions & 28 deletions linkage-monitor/src/test/resources/testproject/gax-bom/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,34 +14,6 @@
<artifactId>gax</artifactId>
<version>1.60.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax</artifactId>
<version>1.60.2-SNAPSHOT</version>
<classifier>testlib</classifier>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
<version>1.60.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-grpc</artifactId>
<version>1.60.2-SNAPSHOT</version>
<classifier>testlib</classifier>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-httpjson</artifactId>
<version>0.77.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax-httpjson</artifactId>
<version>0.77.2-SNAPSHOT</version>
<classifier>testlib</classifier>
</dependency>
</dependencies>
</dependencyManagement>
</project>
3 changes: 3 additions & 0 deletions linkage-monitor/src/test/resources/testproject/linkage-monitor-artifacts-invalid-format.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
com.google.api:gax:1.0.0
com.google.api:gax-grpc:1.0.0
com.google.api:gax-httpjson:1.0.0
2 changes: 2 additions & 0 deletions ...itor/src/test/resources/testproject/linkage-monitor-artifacts-nonexistent-coordinates.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
com.google.api:gax-nonexistent-coordinates
com.google.api:gax-grpc
3 changes: 3 additions & 0 deletions linkage-monitor/src/test/resources/testproject/linkage-monitor-artifacts.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
com.google.api:gax
com.google.api:gax-grpc
com.google.api:gax-httpjson
Comment on lines +1 to +3

@suztomo suztomo Feb 26, 2021

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the file in gax-java repository will look like.