consolidate to single file per package

build.gradle

@@ -29,6 +29,7 @@ buildscript {
         classpath 'com.netflix.nebula:nebula-dependency-recommender:5.2.0'
         classpath 'com.palantir.baseline:gradle-baseline-java:0.19.0'
         classpath 'com.palantir.configurationresolver:gradle-configuration-resolver-plugin:0.3.0'
+        classpath 'com.palantir.gradle.conjure:gradle-conjure:4.2.0'
         classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.10.0'
         classpath 'gradle.plugin.com.palantir:gradle-circle-style:1.1.2'
         classpath 'gradle.plugin.org.inferred:gradle-processors:1.2.15'

conjure-python-core/src/main/java/com/palantir/conjure/python/ConjurePythonGenerator.java

@@ -16,40 +16,35 @@
 
 package com.palantir.conjure.python;
 
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.palantir.conjure.python.client.ClientGenerator;
 import com.palantir.conjure.python.poet.PythonAll;
 import com.palantir.conjure.python.poet.PythonClass;
-import com.palantir.conjure.python.poet.PythonClassName;
 import com.palantir.conjure.python.poet.PythonFile;
-import com.palantir.conjure.python.poet.PythonImport;
 import com.palantir.conjure.python.poet.PythonLine;
 import com.palantir.conjure.python.poet.PythonMetaYaml;
 import com.palantir.conjure.python.poet.PythonSetup;
-import com.palantir.conjure.python.util.TypeNameVisitor;
+import com.palantir.conjure.python.types.PythonBeanGenerator;
 import com.palantir.conjure.spec.ConjureDefinition;
-import com.palantir.conjure.spec.ServiceDefinition;
 import com.palantir.conjure.spec.TypeDefinition;
-import com.palantir.conjure.spec.TypeName;
-import java.util.HashMap;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public final class ConjurePythonGenerator {
 
-    private PythonFileGenerator<TypeDefinition> beanGenerator;
-    private PythonFileGenerator<ServiceDefinition> serviceGenerator;
+    private final PythonBeanGenerator beanGenerator;
+    private final ClientGenerator clientGenerator;
     private final GeneratorConfiguration config;
 
-    public ConjurePythonGenerator(
-            PythonFileGenerator<TypeDefinition> beanGenerator,
-            PythonFileGenerator<ServiceDefinition> serviceGenerator,
+    public ConjurePythonGenerator(PythonBeanGenerator beanGenerator, ClientGenerator clientGenerator,
             GeneratorConfiguration config) {
         this.beanGenerator = beanGenerator;
-        this.serviceGenerator = serviceGenerator;
+        this.clientGenerator = clientGenerator;
         this.config = config;
     }
 
@@ -62,10 +57,7 @@ public void write(ConjureDefinition conjureDefinition, PythonFileWriter writer)
     }
 
     public List<PythonFile> generate(ConjureDefinition conjureDefinition) {
-        Map<TypeName, TypeDefinition> knownTypes = new HashMap<>();
-        TypeDefinition.Visitor<TypeName> indexingVisitor = new TypeNameVisitor();
-        conjureDefinition.getTypes()
-                .forEach(typeDefinition -> knownTypes.put(typeDefinition.accept(indexingVisitor), typeDefinition));
+        List<TypeDefinition> types = conjureDefinition.getTypes();
 
         String pythonicPackageName = config.packageName().replace('-', '_');
         PackageNameProcessor packageNameProcessor = PackageNameProcessor.builder()
@@ -74,70 +66,81 @@ public List<PythonFile> generate(ConjureDefinition conjureDefinition) {
                 .addProcessors(new TopLevelAddingPackageNameProcessor(pythonicPackageName))
                 .build();
 
-        List<PythonFile> beanClasses = knownTypes.values()
+        List<PythonClass> beanClasses = types
                 .stream()
-                .map(objectDefinition -> beanGenerator.generateFile(knownTypes, packageNameProcessor, objectDefinition))
+                .map(objectDefinition -> beanGenerator.generateObject(types, packageNameProcessor, objectDefinition))
                 .collect(Collectors.toList());
 
-        List<PythonFile> serviceClasses = conjureDefinition.getServices()
+        List<PythonClass> serviceClasses = conjureDefinition.getServices()
                 .stream()
-                .map(serviceDef -> serviceGenerator.generateFile(knownTypes, packageNameProcessor, serviceDef))
+                .map(serviceDef -> clientGenerator.generateClient(types, packageNameProcessor, serviceDef))
                 .collect(Collectors.toList());
 
-        Map<String, List<PythonFile>> filesByPackageName = Stream.concat(beanClasses.stream(), serviceClasses.stream())
-                .collect(Collectors.groupingBy(PythonFile::packageName));
+        Map<String, List<PythonClass>> classesByPackageName =
+                Stream.concat(beanClasses.stream(), serviceClasses.stream())
+                        .collect(Collectors.groupingBy(PythonClass::packageName));
 
-        List<PythonFile> moduleInitFiles = filesByPackageName.entrySet()
+        // group into files
+        List<PythonFile.Builder> pythonFiles = classesByPackageName.entrySet()
                 .stream()
-                .map(entry -> buildModuleInit(entry.getKey(), entry.getValue()))
+                .map(entry -> PythonFile.builder()
+                        .packageName(entry.getKey())
+                        .addAllImports(entry.getValue()
+                                .stream()
+                                .flatMap(pt -> pt.requiredImports().stream())
+                                .collect(Collectors.toSet()))
+                        .addAllContents(entry.getValue()))
                 .collect(Collectors.toList());
 
-        List<PythonFile> rootInit = ImmutableList.of(buildRootInit(pythonicPackageName, filesByPackageName.keySet()));
+        Map<Path, Set<Path>> initDefinitions = Maps.newHashMap();
+        pythonFiles.stream()
+                .flatMap(f -> getIntermediateInitPaths(f.build()).entrySet().stream())
+                .forEach(e -> initDefinitions.merge(e.getKey(), e.getValue(), (v1, v2) -> {
+                    Set<Path> combined = Sets.newHashSet(v1);
+                    combined.addAll(v2);
+                    return combined;
+                }));
+        List<PythonFile.Builder> initFiles = initDefinitions.entrySet().stream()
+                .map(e -> buildInitPythonFile(e.getKey(), e.getValue()))
+                .filter(f -> !classesByPackageName.keySet().contains(f.build().packageName()))
+                .collect(Collectors.toList());
 
-        return Stream.of(
-                beanClasses.stream(),
-                serviceClasses.stream(),
-                moduleInitFiles.stream(),
-                rootInit.stream())
-                .flatMap(Function.identity())
+        return Stream.concat(pythonFiles.stream(), initFiles.stream())
+                .map(f -> {
+                    if (f.build().packageName().indexOf('.') < 0) {
+                        f.addContents(versionAttribute(f.build().packageName()));
+                    }
+                    return f;
+                })
+                .map(PythonFile.Builder::build)
                 .collect(Collectors.toList());
     }
 
-    private PythonFile buildModuleInit(String packageName, List<PythonFile> moduleFiles) {
-        return PythonFile.builder()
+    private static Map<Path, Set<Path>> getIntermediateInitPaths(PythonFile pythonFile) {
+        Path filePath = PythonFileWriter.getPath(pythonFile).getParent();
+        Map<Path, Set<Path>> initFiles = Maps.newHashMap();
+        while (filePath.getParent() != null) {
+            initFiles.put(filePath.getParent(), Sets.newHashSet(filePath.getFileName()));
+            filePath = filePath.getParent();
+        }
+        return initFiles;
+    }
+
+    private PythonFile.Builder buildInitPythonFile(Path module, Set<Path> submodules) {
+        String packageName = module.toString().replace('/', '.');
+        PythonAll all = PythonAll.builder()
                 .packageName(packageName)
-                .fileName("__init__.py")
-                .imports(moduleFiles.stream()
-                        .flatMap(file -> file.contents().stream()
-                                .map(pythonClass -> PythonImport.of(
-                                        PythonClassName.of(
-                                                // File has a file extension .py
-                                                "." + file.fileName().substring(0, file.fileName().length() - 3),
-                                                pythonClass.className()))))
-                        .collect(Collectors.toList()))
-                .addContents(PythonAll.builder()
-                        .addAllContents(moduleFiles.stream()
-                                .flatMap(file -> file.contents().stream())
-                                .map(PythonClass::className)
-                                .sorted()
-                                .collect(Collectors.toList()))
-                        .build())
+                .addAllContents(submodules.stream().map(m -> m.toString()).sorted().collect(Collectors.toList()))
                 .build();
+        return PythonFile.builder()
+                .packageName(packageName)
+                .addContents(all);
     }
 
-    private PythonFile buildRootInit(String packageName, Set<String> submodules) {
-        return PythonFile.builder()
+    private PythonClass versionAttribute(String packageName) {
+        return PythonLine.builder()
+                .text(String.format("__version__ = \"%s\"", config.packageVersion()))
                 .packageName(packageName)
-                .fileName("__init__.py")
-                .addContents(PythonAll.builder()
-                        .addAllContents(submodules.stream()
-                                .map(packagePath -> packagePath.substring(packageName.length() + 1))
-                                .sorted()
-                                .collect(Collectors.toList()))
-                        .build())
-                .addContents(PythonLine.builder()
-                        .text(String.format("__version__ = \"%s\"", config.packageVersion()))
-                        .build())
                 .build();
     }
 
@@ -155,6 +158,7 @@ private PythonFile buildPythonSetupFile() {
 
         return PythonFile.builder()
                 .fileName("setup.py")
+                .addAllImports(setup.requiredImports())
                 .addContents(setup)
                 .build();
     }

conjure-python-core/src/main/java/com/palantir/conjure/python/service/ServiceGeneratorPython.java → conjure-python-core/src/main/java/com/palantir/conjure/python/client/ClientGenerator.java

@@ -14,53 +14,50 @@
  * limitations under the License.
  */
 
-package com.palantir.conjure.python.service;
+package com.palantir.conjure.python.client;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet.Builder;
 import com.palantir.conjure.python.PackageNameProcessor;
-import com.palantir.conjure.python.PythonFileGenerator;
+import com.palantir.conjure.python.poet.PythonClass;
+import com.palantir.conjure.python.poet.PythonClassName;
 import com.palantir.conjure.python.poet.PythonEndpointDefinition;
 import com.palantir.conjure.python.poet.PythonEndpointDefinition.PythonEndpointParam;
-import com.palantir.conjure.python.poet.PythonFile;
 import com.palantir.conjure.python.poet.PythonImport;
 import com.palantir.conjure.python.poet.PythonService;
 import com.palantir.conjure.python.types.DefaultTypeNameVisitor;
 import com.palantir.conjure.python.types.MyPyTypeNameVisitor;
+import com.palantir.conjure.python.types.ReferencedTypeNameVisitor;
 import com.palantir.conjure.python.types.TypeMapper;
 import com.palantir.conjure.python.util.CaseConverter;
-import com.palantir.conjure.python.util.ImportsVisitor;
 import com.palantir.conjure.spec.PrimitiveType;
 import com.palantir.conjure.spec.ServiceDefinition;
-import com.palantir.conjure.spec.Type;
 import com.palantir.conjure.spec.TypeDefinition;
-import com.palantir.conjure.spec.TypeName;
 import com.palantir.conjure.visitor.TypeVisitor;
 import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
-public final class ServiceGeneratorPython implements PythonFileGenerator<ServiceDefinition> {
+public final class ClientGenerator {
 
-    @Override
-    public PythonFile generateFile(
-            Map<TypeName, TypeDefinition> types,
-            PackageNameProcessor packageNameProcessor,
+    public PythonClass generateClient(
+            List<TypeDefinition> types,
+            PackageNameProcessor packageNameProvider,
             ServiceDefinition serviceDefinition) {
 
-        ImportsVisitor importsVisitor = new ImportsVisitor(
-                serviceDefinition.getServiceName(), packageNameProcessor, types);
-        TypeMapper mapper = new TypeMapper(new DefaultTypeNameVisitor(types.keySet()));
-        TypeMapper myPyMapper = new TypeMapper(new MyPyTypeNameVisitor(types.keySet()));
-        String packageName = packageNameProcessor.getPackageName(serviceDefinition.getServiceName().getPackage());
+        TypeMapper mapper = new TypeMapper(new DefaultTypeNameVisitor(types));
+        TypeMapper myPyMapper = new TypeMapper(new MyPyTypeNameVisitor(types));
+        ReferencedTypeNameVisitor referencedTypeNameVisitor = new ReferencedTypeNameVisitor(types, packageNameProvider);
 
-        Builder<Type> referencedTypesBuilder = ImmutableSet.builder();
+        Builder<PythonClassName> referencedTypesBuilder = ImmutableSet.builder();
 
         List<PythonEndpointDefinition> endpoints = serviceDefinition.getEndpoints()
                 .stream()
                 .map(ed -> {
-                    ed.getReturns().ifPresent(referencedTypesBuilder::add);
-                    ed.getArgs().forEach(arg -> referencedTypesBuilder.add(arg.getType()));
+                    ed.getReturns()
+                            .ifPresent(returnType -> referencedTypesBuilder.addAll(
+                                    returnType.accept(referencedTypeNameVisitor)));
+                    ed.getArgs().forEach(arg -> referencedTypesBuilder.addAll(
+                            arg.getType().accept(referencedTypeNameVisitor)));
 
                     List<PythonEndpointParam> params = ed.getArgs()
                             .stream()
@@ -94,20 +91,21 @@ public PythonFile generateFile(
                 })
                 .collect(Collectors.toList());
 
+        String packageName =
+                packageNameProvider.getPackageName(serviceDefinition.getServiceName().getPackage());
         List<PythonImport> imports = referencedTypesBuilder.build()
                 .stream()
-                .flatMap(entry -> entry.accept(importsVisitor).stream())
+                .filter(entry -> !entry.conjurePackage().equals(packageName)) // don't need to import if in this file
+                .map(className -> PythonImport.of(className, packageName))
                 .collect(Collectors.toList());
 
-        return PythonFile.builder()
-                .fileName(String.format("%s.py", serviceDefinition.getServiceName().getName()))
+        return PythonService.builder()
                 .packageName(packageName)
-                .imports(imports)
-                .addContents(PythonService.builder()
-                        .className(serviceDefinition.getServiceName().getName())
-                        .docs(serviceDefinition.getDocs())
-                        .addAllEndpointDefinitions(endpoints)
-                        .build())
+                .addAllRequiredImports(PythonService.DEFAULT_IMPORTS)
+                .addAllRequiredImports(imports)
+                .className(serviceDefinition.getServiceName().getName())
+                .docs(serviceDefinition.getDocs())
+                .addAllEndpointDefinitions(endpoints)
                 .build();
     }
 }

conjure-python-core/src/main/java/com/palantir/conjure/python/poet/PythonAll.java

@@ -12,17 +12,16 @@ public interface PythonAll extends PythonClass {
     List<String> contents();
 
     @Override
-    @Value.Default
     default String className() {
-        return "";
+        return "__all__";
     }
 
     @Override
     default void emit(PythonPoetWriter poetWriter) {
         poetWriter.maintainingIndent(() -> {
             poetWriter.writeIndentedLine("__all__ = [");
             poetWriter.increaseIndent();
-            contents().stream().forEach(a -> poetWriter.writeIndentedLine("'%s',", a));
+            contents().forEach(a -> poetWriter.writeIndentedLine("'%s',", a));
             poetWriter.decreaseIndent();
             poetWriter.writeIndentedLine("]");
             poetWriter.writeLine();

conjure-python-core/src/main/java/com/palantir/conjure/python/poet/PythonBean.java

@@ -29,15 +29,21 @@
 public interface PythonBean extends PythonClass {
 
     ImmutableSet<PythonImport> DEFAULT_IMPORTS = ImmutableSet.of(
-            PythonImport.of(PythonClassName.of("conjure_python_client", "ConjureBeanType")),
-            PythonImport.of(PythonClassName.of("conjure_python_client", "ConjureFieldDefinition")));
+            PythonImport.of(PythonClassName.of("typing", "List")),
+            PythonImport.of(PythonClassName.of("typing", "Set")),
+            PythonImport.of(PythonClassName.of("typing", "Dict")),
+            PythonImport.of(PythonClassName.of("typing", "Tuple")),
+            PythonImport.of(PythonClassName.of("typing", "Optional")),
+            PythonImport.of(PythonClassName.of("conjure_python_client", "*")));
 
     @Override
     @Value.Default
     default Set<PythonImport> requiredImports() {
         return DEFAULT_IMPORTS;
     }
 
+    Optional<Documentation> docs();
+
     List<PythonField> fields();
 
     @Override

conjure-python-core/src/main/java/com/palantir/conjure/python/poet/PythonClass.java

@@ -16,16 +16,14 @@
 
 package com.palantir.conjure.python.poet;
 
-import com.palantir.conjure.spec.Documentation;
-import java.util.Optional;
 import java.util.Set;
 
 public interface PythonClass extends Emittable {
 
-    String className();
-
     Set<PythonImport> requiredImports();
 
-    Optional<Documentation> docs();
+    String className();
+
+    String packageName();
 
 }