import groovy.json.JsonOutput import java.util.regex.Pattern import java.util.regex.Matcher import org.gradle.util.GradleVersion // Snyk dependency resolution script for Gradle. // Tested on Gradle versions from v2.14 to v6.8.1 // This script does the following: for all the projects in the build file, // generate a merged configuration of all the available configurations, // and then list the dependencies as a tree. // It's the responsibility of the caller to pick the project(s) they are // interested in from the results. // CLI usages: // gradle -q -I init.gradle snykResolvedDepsJson // gradle -q -I init.gradle snykResolvedDepsJson -Pconfiguration=specificConf -PonlySubProject=sub-project // gradle -q -I init.gradle snykResolvedDepsJson -Pconfiguration=confNameRegex -PconfAttr=buildtype:debug,usage:java-runtime // (-q to have clean output, -P supplies args as per https://stackoverflow.com/a/48370451) // confAttr parameter (supported only in Gradle 3+) is used to perform attribute-based dependency variant matching // (important for Android: https://developer.android.com/studio/build/dependencies#variant_aware) // Its value is a comma-separated list of key:value pairs. The "key" is a case-insensitive substring // of the class name of the attribute (e.g. "buildtype" would match com.android.build.api.attributes.BuildTypeAttr), // the value should be a case-insensitive stringified value of the attribute // Output format: // // Since Gradle is chatty and often prints a "Welcome" banner even with -q option, // the only output lines that matter are: // - prefixed "SNYKECHO ": should be immediately printed as debug information by the caller // - prefixed "JSONDEPS ": JSON representation of the dependencies trees for all projects in the following format // interface JsonDepsScriptResult { // defaultProject: string; // projects: ProjectsDict; // allSubProjectNames: string[]; // } // interface ProjectsDict { // [project: string]: GradleProjectInfo; // } // interface GradleProjectInfo { // depGraph: DepGraph; // snykGraph: SnykGraph; // targetFile: string; // } // interface SnykGraph { // [id: string]: { // name: string; // version: string; // parentIds: string[]; // }; // } class SnykGraph { def nodes def rootId SnykGraph(rootId) { this.nodes = [:] this.rootId = rootId } def setNode(key, value) { if (!key) { return } if (this.nodes.get(key)) { return this.nodes.get(key) } if (!value) { return } def vertex = ['name': value.name, 'version': value.version, 'parentIds': [] as Set] this.nodes.put(key, vertex) return vertex } def setEdge(parentId, childId) { if (!parentId || !childId || parentId == childId) { return } // root-node will be the graphlib root that first-level deps will be attached to if (parentId != this.rootId) { def parentNode = this.setNode(parentId, null) if (!parentNode) { return } } def childNode = this.setNode(childId, null) if (!childNode || childNode.parentIds.contains(parentId)) { return } childNode.parentIds.add(parentId) } } def loadGraph(Iterable deps, SnykGraph graph, parentId, currentChain) { deps.each { dep -> dep.each { d -> def childId = "${d.moduleGroup}:${d.moduleName}@${d.moduleVersion}" if (!graph.nodes.get(childId)) { def childDependency = ['name': "${d.moduleGroup}:${d.moduleName}", 'version': d.moduleVersion] graph.setNode(childId, childDependency) } // In Gradle 2, there can be several instances of the same dependency present at each level, // each for a different configuration. In this case, we need to merge the dependencies. if (!currentChain.contains(childId) && d.children) { currentChain.add(childId) loadGraph(d.children, graph, childId, currentChain) } graph.setEdge(parentId, childId) } } } def getSnykGraph(Iterable deps) { def rootId = 'root-node' def graph = new SnykGraph(rootId) def currentChain = new HashSet() loadGraph(deps, graph, rootId, currentChain) return graph.nodes } def debugLog(msg) { def debug = System.getenv('DEBUG') ?: '' if (debug.length() > 0) { println("SNYKECHO $msg") } } def matchesAttributeFilter(conf, confAttrSpec) { if (!conf.hasProperty('attributes')) { // Gradle before version 3 does not support attributes return true } def matches = true def attrs = conf.attributes attrs.keySet().each({ attr -> def attrValueAsString = attrs.getAttribute(attr).toString().toLowerCase() confAttrSpec.each({ keyValueFilter -> // attr.name is a class name, e.g. com.android.build.api.attributes.BuildTypeAttr if (attr.name.toLowerCase().contains(keyValueFilter[0]) && attrValueAsString != keyValueFilter[1]) { matches = false } }) }) return matches } def findMatchingConfigs(confs, confNameFilter, confAttrSpec) { def matching = confs.findAll({ it.name =~ confNameFilter }) if (matching.isEmpty() && !confs.isEmpty()) { throw new RuntimeException('Matching configurations ' + confNameFilter + ' were not found') } if (confAttrSpec == null) { // We don't have an attribute spec to match return matching } return matching.findAll({ matchesAttributeFilter(it, confAttrSpec) }) } def findProjectConfigs(proj, confNameFilter, confAttrSpec) { def matching = findMatchingConfigs(proj.configurations, confNameFilter, confAttrSpec) if (GradleVersion.current() < GradleVersion.version('3.0')) { proj.configurations.each({ debugLog("conf.name=$it.name") }) return matching } proj.configurations.each({ debugLog("conf.name=$it.name; conf.canBeResolved=$it.canBeResolved; conf.canBeConsumed=$it.canBeConsumed") }) // We are looking for a configuration that `canBeResolved`, because it's a configuration for which // we can compute a dependency graph and that contains all the necessary information for resolution to happen. // See Gradle docs: https://docs.gradle.org/current/userguide/declaring_dependencies.html#sec:resolvable-consumable-configs def resolvable = [] matching.each({ it -> if (!it.canBeResolved) { return } try { // Try accessing resolvedConfiguration to filter out configs that may cause issues in strict lock mode it.resolvedConfiguration resolvable.add(it) } catch (Exception ex) { // Swallow the error debugLog("Skipping config ${it.name} due to resolvedConfiguration error.") } }) debugLog("resolvableConfigs=$resolvable") return resolvable } // We are attaching this task to every project, as this is the only reliable way to run it // when we start with a subproject build.gradle. As a consequence, we need to make sure we // only ever run it once, for the "starting" project. def snykDepsConfExecuted = false allprojects { currProj -> debugLog("Current project: $currProj.name") task snykResolvedDepsJson { def onlyProj = project.hasProperty('onlySubProject') ? onlySubProject : null def confNameFilter = (project.hasProperty('configuration') ? Pattern.compile(configuration, Pattern.CASE_INSENSITIVE) : /.*/ ) def confAttrSpec = (project.hasProperty('confAttr') ? confAttr.toLowerCase().split(',').collect { it.split(':') } : null ) doLast { task -> if (snykDepsConfExecuted) { return } debugLog('snykResolvedDepsJson task is executing via doLast') // debugLog("onlyProj=$onlyProj; confNameFilter=$confNameFilter; confAttrSpec=$confAttrSpec") // First pass: scan all configurations that match the attribute filter and collect all attributes // from them, to use unambiguous values of the attributes on the merged configuration. // // Why we need to scan all sub-projects: if a project A depends on B, and only B has some // configurations with attribute C, we still might need attribute C in our configuration // when resolving the project A, so that it selects a concrete variant of dependency B. def allConfigurationAttributes = [:] // Map, Set> def attributesAsStrings = [:] // Map> rootProject.allprojects.each { proj -> findMatchingConfigs(proj.configurations, confNameFilter, confAttrSpec) .each { conf -> if (!conf.hasProperty('attributes')) { // Gradle before version 3 does not support attributes return } def attrs = conf.attributes attrs.keySet().each({ attr -> def value = attrs.getAttribute(attr) if (!allConfigurationAttributes.containsKey(attr)) { allConfigurationAttributes[attr] = new HashSet() attributesAsStrings[attr.name] = new HashSet() } allConfigurationAttributes[attr].add(value) attributesAsStrings[attr.name].add(value.toString()) }) } } def defaultProjectName = task.project.name def allSubProjectNames = [] def seenSubprojects = [:] allprojects .findAll({ it.name != defaultProjectName }) .each({ def projKey = it.name if (seenSubprojects.get(projKey)) { projKey = it.path.replace(':', '/').replaceAll(~/(^\/+?)|(\/+$)/, '') if (projKey == "") { projKey = defaultProjectName } } allSubProjectNames.add(projKey) seenSubprojects[projKey] = true }) def shouldScanProject = { onlyProj == null || (onlyProj == '.' && it.name == defaultProjectName) || it.name == onlyProj } def projectsDict = [:] debugLog("defaultProjectName=$defaultProjectName; allSubProjectNames=$allSubProjectNames") // These will be used to suggest attribute filtering to the user if the scan fails // due to ambiguous resolution of dependency variants def jsonAttrs = JsonOutput.toJson(attributesAsStrings) println("JSONATTRS $jsonAttrs") rootProject.allprojects.findAll(shouldScanProject).each { proj -> debugLog("processing project: name=$proj.name; path=$proj.path") def resolvableConfigs = findProjectConfigs(proj, confNameFilter, confAttrSpec) def resolvedConfigs = [] resolvableConfigs.each({ config -> def resConf = config.resolvedConfiguration debugLog("config `$config.name' resolution has errors: ${resConf.hasError()}") if (!resConf.hasError()) { resolvedConfigs.add(resConf) debugLog("first level module dependencies for config `$config.name': $resConf.firstLevelModuleDependencies") } }) if (resolvedConfigs.isEmpty() && !resolvableConfigs.isEmpty()) { throw new RuntimeException('Configurations: ' + resolvableConfigs.collect { it.name } + ' for project ' + proj + ' could not be resolved.') } def nonemptyFirstLevelDeps = resolvedConfigs.resolvedConfiguration.firstLevelModuleDependencies.findAll({ !it.isEmpty() }) debugLog("non-empty first level deps for project `$proj.name': $nonemptyFirstLevelDeps") debugLog('converting gradle graph to snyk-graph format') def projGraph = getSnykGraph(nonemptyFirstLevelDeps) def projKey = proj.name if (projectsDict.get(projKey)) { debugLog("The deps dict already has a project with key `$projKey'; using the full path") projKey = proj.path.replace(':', '/').replaceAll(~/(^\/+?)|(\/+$)/, '') if (projKey == "") { debugLog("project path is empty (proj.path=$proj.path)! will use defaultProjectName=$defaultProjectName") projKey = defaultProjectName } } projectsDict[projKey] = [ 'targetFile': findProject(proj.path).buildFile.toString(), 'snykGraph': projGraph, 'projectVersion': proj.version.toString() ] } def result = [ 'defaultProject': defaultProjectName, 'projects': projectsDict, 'allSubProjectNames': allSubProjectNames ] def jsonDeps = JsonOutput.toJson(result) println("JSONDEPS $jsonDeps") snykDepsConfExecuted = true } } }