Vendetta: single-file and multi-location deps

Changelog.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+## 3.17.0
+
+- Vendetta: Support single-file library dependencies and multi-location vendored dependencies. Locations are now correctly classified as files or directories in vendored metadata. ([#1680](https://github.com/fossas/fossa-cli/pull/1680))
 - Windows: Fix UTF-8 encoding crash when running `fossa analyze --x-vendetta --debug` on native Windows. The CLI no longer crashes when subprocess output contains Unicode characters. ([#1646](https://github.com/fossas/fossa-cli/pull/1646))
 
 ## 3.16.5

src/App/Fossa/Ficus/Analyze.hs

@@ -6,6 +6,7 @@ module App.Fossa.Ficus.Analyze (
   analyzeWithFicusMain,
   -- Exported for testing
   singletonFicusMessage,
+  vendoredDepsToSourceUnit,
 )
 where
 
@@ -57,6 +58,7 @@ import Fossa.API.Types (ApiKey (..), ApiOpts (..))
 import Path (Abs, Dir, Path, toFilePath)
 import Prettyprinter (pretty)
 import Srclib.Types (Locator (..), SourceUnit (..), SourceUnitBuild (..), SourceUnitDependency (..), renderLocator, textToOriginPath)
+import System.Directory qualified as Directory
 import System.FilePath ((</>))
 import System.IO (Handle, IOMode (WriteMode), hClose, hPutStrLn, hSetEncoding, openFile, stderr, utf8)
 import System.Process.Typed (
@@ -181,33 +183,32 @@ findingToVendoredDependency (FicusFinding (FicusMessageData strategy payload))
       decode (BL.fromStrict $ Text.Encoding.encodeUtf8 payload)
 findingToVendoredDependency _ = Nothing
 
-vendoredDepsToSourceUnit :: [FicusVendoredDependency] -> SourceUnit
-vendoredDepsToSourceUnit deps =
-  SourceUnit
-    { sourceUnitName = "ficus-vendored-dependencies"
-    , sourceUnitType = "ficus-vendored"
-    , sourceUnitManifest = "ficus-vendored-dependencies"
-    , sourceUnitBuild =
-        Just $
-          SourceUnitBuild
-            { buildArtifact = "default"
-            , buildSucceeded = True
-            , buildImports = locators
-            , buildDependencies = dependencies
-            }
-    , sourceUnitGraphBreadth = Complete
-    , sourceUnitNoticeFiles = []
-    , sourceUnitOriginPaths = map (textToOriginPath . ficusVendoredDependencyPath) deps
-    , sourceUnitLabels = Nothing
-    , additionalData = Nothing
-    }
+vendoredDepsToSourceUnit :: FilePath -> [FicusVendoredDependency] -> IO SourceUnit
+vendoredDepsToSourceUnit rootDir deps = do
+  dependencies <- traverse vendoredDepToSourceUnitDependency deps
+  pure
+    SourceUnit
+      { sourceUnitName = "ficus-vendored-dependencies"
+      , sourceUnitType = "ficus-vendored"
+      , sourceUnitManifest = "ficus-vendored-dependencies"
+      , sourceUnitBuild =
+          Just $
+            SourceUnitBuild
+              { buildArtifact = "default"
+              , buildSucceeded = True
+              , buildImports = locators
+              , buildDependencies = dependencies
+              }
+      , sourceUnitGraphBreadth = Complete
+      , sourceUnitNoticeFiles = []
+      , sourceUnitOriginPaths = concatMap (map textToOriginPath . ficusVendoredDependencyLocations) deps
+      , sourceUnitLabels = Nothing
+      , additionalData = Nothing
+      }
   where
     locators :: [Locator]
     locators = map vendoredDepToLocator deps
 
-    dependencies :: [SourceUnitDependency]
-    dependencies = map vendoredDepToSourceUnitDependency deps
-
     vendoredDepToLocator :: FicusVendoredDependency -> Locator
     vendoredDepToLocator dep =
       Locator
@@ -216,21 +217,29 @@ vendoredDepsToSourceUnit deps =
         , locatorRevision = ficusVendoredDependencyVersion dep
         }
 
-    vendoredDepToSourceUnitDependency :: FicusVendoredDependency -> SourceUnitDependency
-    vendoredDepToSourceUnitDependency dep =
-      SourceUnitDependency
-        { sourceDepLocator = vendoredDepToLocator dep
-        , sourceDepImports = []
-        , sourceDepData =
-            Aeson.object
-              [ "vendored"
-                  Aeson..= [ Aeson.object
-                               [ "type" Aeson..= ("directory" :: Text)
-                               , "path" Aeson..= ficusVendoredDependencyPath dep
-                               ]
-                           ]
-              ]
-        }
+    vendoredDepToSourceUnitDependency :: FicusVendoredDependency -> IO SourceUnitDependency
+    vendoredDepToSourceUnitDependency dep = do
+      vendoredEntries <- traverse classifyLocation (ficusVendoredDependencyLocations dep)
+      pure
+        SourceUnitDependency
+          { sourceDepLocator = vendoredDepToLocator dep
+          , sourceDepImports = []
+          , sourceDepData =
+              Aeson.object
+                [ "vendored" Aeson..= vendoredEntries
+                ]
+          }
+
+    classifyLocation :: Text -> IO Aeson.Value
+    classifyLocation loc = do
+      let fullPath = rootDir </> toString loc
+      isFile <- Directory.doesFileExist fullPath
+      let locType = if isFile then "file" :: Text else "directory"
+      pure $
+        Aeson.object
+          [ "type" Aeson..= locType
+          , "path" Aeson..= loc
+          ]
 
 runFicus ::
   ( Has Diagnostics sig m
@@ -346,9 +355,12 @@ runFicus maybeDebugDir ficusConfig = do
               (Nothing, [])
 
       let (snippetResults, vendoredDeps) = accumulator
-      let vendoredResults = case vendoredDeps of
-            [] -> Nothing
-            deps -> Just $ FicusVendoredDependencyScanResults (Just $ vendoredDepsToSourceUnit deps)
+      let rootDir = toFilePath $ ficusConfigRootDir ficusConfig
+      vendoredResults <- case vendoredDeps of
+        [] -> pure Nothing
+        deps -> do
+          srcUnit <- vendoredDepsToSourceUnit rootDir deps
+          pure . Just $ FicusVendoredDependencyScanResults (Just srcUnit)
 
       pure $
         FicusAnalysisResults

src/App/Fossa/Ficus/Types.hs

@@ -44,7 +44,7 @@ data FicusVendoredDependency = FicusVendoredDependency
   { ficusVendoredDependencyName :: Text
   , ficusVendoredDependencyEcosystem :: Text
   , ficusVendoredDependencyVersion :: Maybe Text
-  , ficusVendoredDependencyPath :: Text
+  , ficusVendoredDependencyLocations :: [Text]
   }
   deriving (Eq, Ord, Show, Generic)
 
@@ -54,7 +54,7 @@ instance FromJSON FicusVendoredDependency where
       <$> obj .: "name"
       <*> obj .: "ecosystem"
       <*> obj .:? "version"
-      <*> obj .: "path"
+      <*> obj .: "locations"
 
 data FicusSnippetScanResults = FicusSnippetScanResults
   { ficusSnippetScanResultsAnalysisId :: Int

test/Ficus/FicusSpec.hs

@@ -3,17 +3,24 @@
 
 module Ficus.FicusSpec (spec) where
 
-import App.Fossa.Ficus.Analyze (analyzeWithFicus)
-import App.Fossa.Ficus.Types (FicusAnalysisResults (..), FicusSnippetScanResults (..), FicusStrategy (FicusStrategySnippetScan, FicusStrategyVendetta), FicusVendoredDependencyScanResults (FicusVendoredDependencyScanResults))
+import App.Fossa.Ficus.Analyze (analyzeWithFicus, vendoredDepsToSourceUnit)
+import App.Fossa.Ficus.Types (FicusAnalysisResults (..), FicusSnippetScanResults (..), FicusStrategy (FicusStrategySnippetScan, FicusStrategyVendetta), FicusVendoredDependency (..), FicusVendoredDependencyScanResults (FicusVendoredDependencyScanResults))
 import App.Types (ProjectRevision (..))
 import Control.Effect.Lift (sendIO)
 import Control.Timeout (Duration (Seconds))
+import Data.Aeson qualified as Aeson
+import Data.Aeson.Key qualified as Key
+import Data.Aeson.KeyMap qualified as KeyMap
 import Data.String.Conversion (toText)
+import Data.Vector qualified
 import Fossa.API.Types (ApiKey (..), ApiOpts (..))
 import Path (Dir, Path, Rel, reldir, (</>))
 import Path.IO qualified as PIO
-import Srclib.Types (SourceUnit (sourceUnitName))
+import Srclib.Types (SourceUnit (..), SourceUnitBuild (..), SourceUnitDependency (..))
+import System.Directory qualified as Directory
 import System.Environment (lookupEnv)
+import System.FilePath qualified as FP
+import System.Random (randomIO)
 import Test.Effect (expectationFailure', it', shouldBe', shouldSatisfy')
 import Test.Hspec
 import Text.URI (mkURI)
@@ -68,3 +75,52 @@ spec = do
               -- No vendetta results returned - this is acceptable for integration testing
               True `shouldBe'` True
         _ -> expectationFailure' "Ficus analysis returned no results unexpectedly."
+
+  describe "vendoredDepsToSourceUnit" $ do
+    it "classifies directories and files correctly in vendored metadata" $ do
+      systemTmpDir <- Directory.getTemporaryDirectory
+      suffix <- show <$> (randomIO :: IO Word)
+      let tmpDir = systemTmpDir FP.</> ("ficus-vendored-test-" <> suffix)
+      Directory.createDirectoryIfMissing True (tmpDir FP.</> "vendored")
+      writeFile (tmpDir FP.</> "sqlite3.c") ""
+
+      let dep =
+            FicusVendoredDependency
+              { ficusVendoredDependencyName = "github.com/test/dep"
+              , ficusVendoredDependencyEcosystem = "git"
+              , ficusVendoredDependencyVersion = Nothing
+              , ficusVendoredDependencyLocations = ["vendored", "sqlite3.c"]
+              }
+
+      result <- vendoredDepsToSourceUnit tmpDir [dep]
+
+      Directory.removeDirectoryRecursive tmpDir
+
+      sourceUnitName result `shouldBe` "ficus-vendored-dependencies"
+
+      -- Check origin paths contain all locations
+      length (sourceUnitOriginPaths result) `shouldBe` 2
+
+      -- Check the vendored metadata classifies types correctly
+      case sourceUnitBuild result of
+        Nothing -> expectationFailure "Expected SourceUnitBuild"
+        Just build -> case buildDependencies build of
+          [srcDep] -> case sourceDepData srcDep of
+            Aeson.Object obj -> case KeyMap.lookup (Key.fromString "vendored") obj of
+              Just (Aeson.Array entries) -> do
+                length entries `shouldBe` 2
+                -- First entry should be the directory
+                case entries Data.Vector.!? 0 of
+                  Just (Aeson.Object e) -> do
+                    KeyMap.lookup (Key.fromString "type") e `shouldBe` Just (Aeson.String "directory")
+                    KeyMap.lookup (Key.fromString "path") e `shouldBe` Just (Aeson.String "vendored")
+                  _ -> expectationFailure "Expected object in vendored array"
+                -- Second entry should be the file
+                case entries Data.Vector.!? 1 of
+                  Just (Aeson.Object e) -> do
+                    KeyMap.lookup (Key.fromString "type") e `shouldBe` Just (Aeson.String "file")
+                    KeyMap.lookup (Key.fromString "path") e `shouldBe` Just (Aeson.String "sqlite3.c")
+                  _ -> expectationFailure "Expected object in vendored array"
+              _ -> expectationFailure "Expected 'vendored' array in metadata"
+            _ -> expectationFailure "Expected object in sourceDepData"
+          _ -> expectationFailure "Expected exactly one dependency"