Merge changes Ib2e07564,I53c0f977,Ia1499ded into main

* changes:
  Add command to analyze crate imports for problems.
  Add a new command to update TEST_MAPPING files.
  Crate for managing TEST_MAPPING files.

Author: jfgoog

tools/external_crates/Cargo.toml

@@ -6,5 +6,6 @@ members = [
     "name_and_version",
     "name_and_version_proc_macros",
     "rooted_path",
+    "test_mapping",
 ]
 resolver = "2"

tools/external_crates/cargo_embargo.json

@@ -19,6 +19,7 @@
   "workspace": true,
   "workspace_excludes": [
     "crate_tool",
-    "license_checker"
+    "license_checker",
+    "test_mapping"
   ]
 }

tools/external_crates/crate_tool/Cargo.toml

@@ -15,6 +15,7 @@ crates-index = "3.2.0"
 glob = "0.3"
 itertools = "0.11"
 num_cpus = "1"
+owning_ref = "0.4"
 protobuf = "3"
 reqwest = { version = "0.12.5", features = ["blocking", "gzip"] }
 semver = "1"
@@ -31,6 +32,7 @@ license_checker = { path = "../license_checker" }
 name_and_version = { path = "../name_and_version" }
 name_and_version_proc_macros = { path = "../name_and_version_proc_macros" }
 rooted_path = { path = "../rooted_path" }
+test_mapping = { path = "../test_mapping" }
 
 [build-dependencies]
 protobuf-codegen = "3"

tools/external_crates/crate_tool/src/main.rs

@@ -61,6 +61,12 @@ enum Cmd {
         #[command(flatten)]
         crates: MigrationCrateList,
     },
+    /// Analyze a crate to see if it can be imported.
+    #[command(hide = true)]
+    AnalyzeImport {
+        /// The crate name.
+        crate_name: String,
+    },
     /// Import a crate and its dependencies into the monorepo.
     #[command(hide = true)]
     Import {
@@ -118,6 +124,12 @@ enum Cmd {
     TryUpdates {},
     /// Initialize a new managed repo.
     Init {},
+    /// Update TEST_MAPPING files.
+    #[command(hide = true)]
+    TestMapping {
+        #[command(flatten)]
+        crates: CrateList,
+    },
 }
 
 #[derive(Args)]
@@ -137,11 +149,7 @@ struct CrateList {
 impl CrateList {
     fn to_list(&self, managed_repo: &ManagedRepo) -> Result<Vec<String>> {
         Ok(if self.all {
-            managed_repo
-                .all_crate_names()?
-                .into_iter()
-                .filter(|crate_name| !self.exclude.contains(crate_name))
-                .collect::<Vec<_>>()
+            managed_repo.all_crate_names()?.difference(&self.exclude).cloned().collect::<Vec<_>>()
         } else {
             self.crates.clone()
         })
@@ -209,6 +217,7 @@ fn main() -> Result<()> {
             managed_repo.regenerate(crates.to_list(&managed_repo)?.into_iter(), true)
         }
         Cmd::PreuploadCheck { files } => managed_repo.preupload_check(&files),
+        Cmd::AnalyzeImport { crate_name } => managed_repo.analyze_import(&crate_name),
         Cmd::Import { crate_name } => managed_repo.import(&crate_name),
         Cmd::FixLicenses { crates } => {
             managed_repo.fix_licenses(crates.to_list(&managed_repo)?.into_iter())
@@ -225,5 +234,8 @@ fn main() -> Result<()> {
         Cmd::Update { crate_name, version } => managed_repo.update(crate_name, version),
         Cmd::TryUpdates {} => managed_repo.try_updates(),
         Cmd::Init {} => managed_repo.init(),
+        Cmd::TestMapping { crates } => {
+            managed_repo.fix_test_mapping(crates.to_list(&managed_repo)?.into_iter())
+        }
     }
 }

tools/external_crates/crate_tool/src/managed_crate.rs

@@ -27,6 +27,7 @@ use license_checker::find_licenses;
 use name_and_version::NamedAndVersioned;
 use rooted_path::RootedPath;
 use semver::Version;
+use test_mapping::TestMapping;
 
 use crate::{
     android_bp::run_cargo_embargo,
@@ -217,6 +218,14 @@ impl<State: ManagedCrateState> ManagedCrate<State> {
         metadata.write()?;
         Ok(())
     }
+    pub fn fix_test_mapping(&self) -> Result<()> {
+        let mut tm = TestMapping::read(self.android_crate_path().clone())?;
+        println!("{}", self.name());
+        if tm.fix_import_paths() || tm.add_new_tests_to_postsubmit()? {
+            tm.write()?;
+        }
+        Ok(())
+    }
 }
 
 impl ManagedCrate<New> {

tools/external_crates/crate_tool/src/managed_repo.rs

@@ -23,6 +23,7 @@ use std::{
 };
 
 use anyhow::{anyhow, Context, Result};
+use crates_index::DependencyKind;
 use glob::glob;
 use google_metadata::GoogleMetadata;
 use itertools::Itertools;
@@ -105,13 +106,13 @@ impl ManagedRepo {
     ) -> Result<ManagedCrate<crate::managed_crate::New>> {
         Ok(ManagedCrate::new(Crate::from(self.managed_dir_for(crate_name))?))
     }
-    pub fn all_crate_names(&self) -> Result<Vec<String>> {
-        let mut managed_dirs = Vec::new();
+    pub fn all_crate_names(&self) -> Result<BTreeSet<String>> {
+        let mut managed_dirs = BTreeSet::new();
         if self.managed_dir().abs().exists() {
             for entry in read_dir(self.managed_dir())? {
                 let entry = entry?;
                 if entry.path().is_dir() {
-                    managed_dirs.push(entry.file_name().into_string().map_err(|e| {
+                    managed_dirs.insert(entry.file_name().into_string().map_err(|e| {
                         anyhow!("Failed to convert {} to string", e.to_string_lossy())
                     })?);
                 }
@@ -364,6 +365,75 @@ impl ManagedRepo {
 
         Ok(())
     }
+    pub fn analyze_import(&self, crate_name: &str) -> Result<()> {
+        if self.contains(crate_name) {
+            println!("Crate already imported at {}", self.managed_dir_for(crate_name));
+            return Ok(());
+        }
+        let legacy_dir = self.legacy_dir_for(crate_name, None)?;
+        if legacy_dir.abs().exists() {
+            println!("Legacy crate already imported at {}", legacy_dir);
+            return Ok(());
+        }
+
+        let mut managed_crates = self.new_cc();
+        managed_crates.add_from(self.managed_dir().rel())?;
+        let legacy_crates = self.legacy_crates()?;
+
+        let cio_crate = self.crates_io.get_crate(crate_name)?;
+
+        for version in cio_crate.versions() {
+            println!("Version {}", version.version());
+            let mut found_problems = false;
+            for (dep, req) in version.android_deps_with_version_reqs() {
+                let cc = if managed_crates.contains_name(dep.crate_name()) {
+                    &managed_crates
+                } else {
+                    &legacy_crates
+                };
+                if !cc.contains_name(dep.crate_name()) {
+                    found_problems = true;
+                    println!(
+                        "  Dep {} {} has not been imported to Android",
+                        dep.crate_name(),
+                        dep.requirement()
+                    );
+                    if matches!(dep.kind(), DependencyKind::Dev) {
+                        println!("    But this is a dev dependency, probably only needed if you want to run the tests");
+                    }
+                    if dep.is_optional() {
+                        println!("    But this is an optional dependency, used by the following features: {}", dep.features().join(", "));
+                    }
+                    continue;
+                }
+                let versions = cc.get_versions(dep.crate_name()).collect::<Vec<_>>();
+                let has_matching_version =
+                    versions.iter().any(|(nv, _)| req.matches_relaxed(nv.version()));
+                if !has_matching_version {
+                    found_problems = true;
+                }
+                if !has_matching_version || versions.len() > 1 {
+                    if has_matching_version {
+                        println!("  Dep {} has multiple versions available. You may need to override the default choice in cargo_embargo.json", dep.crate_name());
+                    }
+                    for (_, dep_crate) in versions {
+                        println!(
+                            "  Dep {} {} is {}satisfied by v{} at {}",
+                            dep.crate_name(),
+                            dep.requirement(),
+                            if req.matches_relaxed(dep_crate.version()) { "" } else { "not " },
+                            dep_crate.version(),
+                            dep_crate.path()
+                        );
+                    }
+                }
+            }
+            if !found_problems {
+                println!("  No problems found with this version.")
+            }
+        }
+        Ok(())
+    }
     pub fn import(&self, crate_name: &str) -> Result<()> {
         let (new_deps, pseudo_crate) = self.add_crate_and_dependencies(crate_name)?;
 
@@ -503,7 +573,7 @@ impl ManagedRepo {
         let pseudo_crate = self.pseudo_crate().vendor()?;
         let deps = pseudo_crate.deps().keys().cloned().collect::<BTreeSet<_>>();
 
-        let managed_dirs = self.all_crate_names()?.into_iter().collect();
+        let managed_dirs = self.all_crate_names()?;
 
         if deps != managed_dirs {
             return Err(anyhow!("Deps in pseudo_crate/Cargo.toml don't match directories in {}\nDirectories not in Cargo.toml: {}\nCargo.toml deps with no directory: {}",
@@ -837,6 +907,13 @@ impl ManagedRepo {
         self.pseudo_crate().init()?;
         Ok(())
     }
+    pub fn fix_test_mapping<T: AsRef<str>>(&self, crates: impl Iterator<Item = T>) -> Result<()> {
+        for crate_name in crates {
+            let mc = self.managed_crate_for(crate_name.as_ref())?;
+            mc.fix_test_mapping()?;
+        }
+        Ok(())
+    }
 }
 
 // Files that are ignored when migrating a crate to the monorepo.

tools/external_crates/test_mapping/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "test_mapping"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+android_bp = "0.3"
+json-strip-comments = "1"
+owning_ref = "0.4"
+serde = { version = "=1.0.210", features = ["derive"] }
+serde_json = "1"
+thiserror = "1.0"
+rooted_path = { path = "../rooted_path" }