Improvement/use testcontainers for dev

.run/Backend with DB.run.xml

@@ -0,0 +1,25 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Backend with DB" type="Application" factoryName="Application">
+    <option name="MAIN_CLASS_NAME" value="ai.giskard.GiskardTestApp" />
+    <module name="giskard.backend.test" />
+    <extension name="coverage">
+      <pattern>
+        <option name="PATTERN" value="ai.giskard.*" />
+        <option name="ENABLED" value="true" />
+      </pattern>
+    </extension>
+    <extension name="net.ashald.envfile">
+      <option name="IS_ENABLED" value="false" />
+      <option name="IS_SUBST" value="false" />
+      <option name="IS_PATH_MACRO_SUPPORTED" value="false" />
+      <option name="IS_IGNORE_MISSING_FILES" value="false" />
+      <option name="IS_ENABLE_EXPERIMENTAL_INTEGRATIONS" value="false" />
+      <ENTRIES>
+        <ENTRY IS_ENABLED="true" PARSER="runconfig" IS_EXECUTABLE="false" />
+      </ENTRIES>
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>

.run/Backend.run.xml

@@ -1,11 +1,13 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Backend" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
-    <option name="ACTIVE_PROFILES" value="dev" />
-    <envs>
-      <env name="GISKARD_AUTH" value="true" />
-    </envs>
+  <configuration default="false" name="Backend" type="Application" factoryName="Application">
+    <option name="MAIN_CLASS_NAME" value="ai.giskard.GiskardApp" />
     <module name="giskard.backend.main" />
-    <option name="SPRING_BOOT_MAIN_CLASS" value="ai.giskard.GiskardApp" />
+    <extension name="coverage">
+      <pattern>
+        <option name="PATTERN" value="ai.giskard.*" />
+        <option name="ENABLED" value="true" />
+      </pattern>
+    </extension>
     <extension name="net.ashald.envfile">
       <option name="IS_ENABLED" value="false" />
       <option name="IS_SUBST" value="false" />

backend/.gitignore

@@ -154,3 +154,4 @@ Desktop.ini
 ######################
 /coverage/
 /.nyc_output/
+generated

backend/build.gradle.kts

@@ -149,7 +149,7 @@ dependencies {
     annotationProcessor("org.springframework.boot:spring-boot-configuration-processor:${springBootVersion}")
     compileOnly("org.mapstruct:mapstruct:${mapstructVersion}")
     compileOnly("org.projectlombok:lombok")
-    developmentOnly("org.springframework.boot:spring-boot-devtools:${springBootVersion}")
+    testImplementation("org.springframework.boot:spring-boot-devtools:${springBootVersion}")
     implementation("com.fasterxml.jackson.core:jackson-annotations")
     implementation("com.fasterxml.jackson.core:jackson-databind")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-hibernate6")
@@ -187,11 +187,12 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-tomcat")
     implementation("org.springframework.boot:spring-boot-starter-web")
     implementation("org.springframework.boot:spring-boot-starter-websocket")
+    implementation("org.springframework.boot:spring-boot-testcontainers")
     implementation("org.springframework.security:spring-security-config")
     implementation("org.springframework.security:spring-security-data")
     implementation("org.springframework.security:spring-security-messaging")
     implementation("org.springframework.security:spring-security-web")
-    implementation("org.testcontainers:postgresql:1.18.3")
+    implementation("org.testcontainers:postgresql")
     implementation(files("$projectDir/src/main/resources/third-party/j2ts-api.jar"))
     implementation(group = "com.fasterxml.jackson.dataformat", name = "jackson-dataformat-yaml", version = "2.13.1")
     implementation(group = "com.github.luben", name = "zstd-jni", version = "1.5.2-3")

backend/src/main/java/ai/giskard/config/LiquibaseConfiguration.java

@@ -72,7 +72,7 @@ public SpringLiquibase liquibase(
         liquibase.setDatabaseChangeLogLockTable(liquibaseProperties.getDatabaseChangeLogLockTable());
         liquibase.setDatabaseChangeLogTable(liquibaseProperties.getDatabaseChangeLogTable());
         liquibase.setDropFirst(liquibaseProperties.isDropFirst());
-        liquibase.setLabels(liquibaseProperties.getLabels());
+        liquibase.setLabelFilter(liquibaseProperties.getLabelFilter());
         liquibase.setChangeLogParameters(liquibaseProperties.getParameters());
         liquibase.setRollbackFile(liquibaseProperties.getRollbackFile());
         liquibase.setTestRollbackOnUpdate(liquibaseProperties.isTestRollbackOnUpdate());

backend/src/main/java/ai/giskard/config/SecurityConfiguration.java

@@ -12,7 +12,6 @@
 import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
-import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer;
 import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
 import org.springframework.security.config.annotation.web.configurers.HeadersConfigurer;
 import org.springframework.security.config.http.SessionCreationPolicy;
@@ -47,40 +46,40 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
                 .frameOptions(HeadersConfigurer.FrameOptionsConfig::sameOrigin)
             ).sessionManagement(conf -> conf.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
             .authorizeHttpRequests(authorize -> authorize
-                .requestMatchers(antMatcher("/api/v2/dev/**")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/settings/license")).permitAll()
+                .requestMatchers(
+                    antMatcher(WEBSOCKET_ENDPOINT),
+                    antMatcher(MLWORKER_WEBSOCKET_ENDPOINT),
+                    antMatcher(HttpMethod.OPTIONS, "/**"),
+                    antMatcher("/swagger-ui/**"),
+                    antMatcher("/test/**"),
+                    antMatcher("/api/v2/dev/**"),
+                    antMatcher("/api/v2/settings/license"),
+                    antMatcher("/api/v2/settings"),
+                    antMatcher("/api/v2/setup"),
+                    antMatcher("/api/v2/ee/license"),
+                    antMatcher("/api/v2/authenticate"),
+                    antMatcher("/api/v2/register"),
+                    antMatcher("/api/v2/register"),
+                    antMatcher("/api/v2/activate"),
+                    antMatcher("/api/v2/account/password-recovery"),
+                    antMatcher("/api/v2/account/reset-password"),
+                    antMatcher("/management/health"),
+                    antMatcher("/management/health/**"),
+                    antMatcher("/management/info"),
+                    antMatcher("/management/prometheus")
+                ).permitAll()
+                .requestMatchers(
+                    antMatcher("/api/admin/**"),
+                    antMatcher("/management/**")
+                ).hasAuthority(AuthoritiesConstants.ADMIN)
                 .requestMatchers(antMatcher("/api/v2/settings/ml-worker-connect")).hasAuthority(AuthoritiesConstants.API)
-                .requestMatchers(antMatcher("/api/v2/settings")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/setup")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/ee/license")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/authenticate")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/register")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/register")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/activate")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/account/password-recovery")).permitAll()
-                .requestMatchers(antMatcher("/api/v2/account/reset-password")).permitAll()
-                .requestMatchers(antMatcher("/api/admin/**")).hasAuthority(AuthoritiesConstants.ADMIN)
                 .requestMatchers(antMatcher("/api/**")).authenticated()
-                .requestMatchers(antMatcher("/management/health")).permitAll()
-                .requestMatchers(antMatcher("/management/health/**")).permitAll()
-                .requestMatchers(antMatcher("/management/info")).permitAll()
-                .requestMatchers(antMatcher("/management/prometheus")).permitAll()
-                .requestMatchers(antMatcher("/management/**")).hasAuthority(AuthoritiesConstants.ADMIN)
             )
             .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
             .apply(securityConfigurerAdapter());
         return http.build();
     }
 
-    @Bean
-    public WebSecurityCustomizer webSecurityCustomizer() {
-        return web -> web.ignoring()
-            .requestMatchers(antMatcher(WEBSOCKET_ENDPOINT))
-            .requestMatchers(antMatcher(MLWORKER_WEBSOCKET_ENDPOINT))
-            .requestMatchers(antMatcher(HttpMethod.OPTIONS, "/**"))
-            .requestMatchers(antMatcher("/swagger-ui/**"))
-            .requestMatchers(antMatcher("/test/**"));
-    }
 
     @Bean
     public PasswordEncoder passwordEncoder() {

backend/src/main/java/ai/giskard/domain/BaseEntity.java

@@ -8,7 +8,6 @@
 
 @Getter
 @MappedSuperclass
-@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
 public abstract class BaseEntity implements Serializable {
     @Id
     @JsonIgnore

backend/src/main/java/ai/giskard/domain/ml/FunctionInput.java

@@ -27,7 +27,7 @@ public class FunctionInput implements Serializable {
 
     private String type;
 
-    @Column(columnDefinition = "CLOB", name = "input_value")
+    @Column(name = "input_value")
     @Lob
     private String value;
 

backend/src/main/java/ai/giskard/domain/ml/Inspection.java

@@ -3,13 +3,13 @@
 
 import ai.giskard.domain.AbstractAuditingEntity;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.Getter;
-import lombok.Setter;
-
 import jakarta.persistence.Entity;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.Id;
 import jakarta.persistence.ManyToOne;
+import lombok.Getter;
+import lombok.Setter;
+
 import java.io.Serial;
 
 @Getter

backend/src/main/java/ai/giskard/service/InitService.java

@@ -23,12 +23,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.google.common.collect.Sets;
+import jakarta.validation.constraints.NotNull;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.boot.context.event.ApplicationStartedEvent;
 import org.springframework.context.event.EventListener;
 import org.springframework.core.env.Environment;
 import org.springframework.core.io.ClassPathResource;
@@ -38,7 +39,6 @@
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
-import jakarta.validation.constraints.NotNull;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -146,8 +146,9 @@ public String getProjectByCreatorLogin(String login) {
     /**
      * Initializing first authorities, mock users, and mock projects
      */
-    @EventListener(ApplicationReadyEvent.class)
+    @EventListener(ApplicationStartedEvent.class)
     public void init() {
+        logger.info("Initializing Giskard");
         projects = createProjectConfigMap();
         generalSettingsService.saveIfNotExists(new GeneralSettings());
         initAuthorities();
@@ -156,6 +157,7 @@ public void init() {
         if (!profiles.contains("prod") && !profiles.contains("dev")) {
             initProjects();
         }
+        logger.info("DONE Initializing Giskard");
     }
 
     /**

backend/src/main/java/ai/giskard/web/rest/errors/GlobalExceptionHandler.java

@@ -34,6 +34,7 @@ private static HttpStatus getHttpStatus(Throwable ex) {
         if (ex instanceof ConcurrencyFailureException) return HttpStatus.CONFLICT;
         if (ex instanceof BadCredentialsException) return HttpStatus.UNAUTHORIZED;
         if (ex instanceof AccessDeniedException) return HttpStatus.FORBIDDEN;
+        if (ex instanceof IllegalArgumentException) return HttpStatus.BAD_REQUEST;
         return HttpStatus.INTERNAL_SERVER_ERROR;
     }
 }

backend/src/main/resources/config/application-dev.yml

@@ -16,7 +16,7 @@ logging:
     ROOT: INFO
     org.hibernate.SQL: INFO
     ai.giskard: INFO
-    com.zaxxer.hikari.HikariConfig: DEBUG
+    com.zaxxer.hikari.HikariConfig: INFO
     com.zaxxer.hikari: INFO
 
 spring:
@@ -39,9 +39,6 @@ spring:
       minimum-idle: 10
       connection-timeout: 20000
       leak-detection-threshold: 10000
-  jpa:
-    database-platform: org.hibernate.dialect.H2Dialect
-    show-sql: false
   liquibase:
     # Remove 'faker' if you do not want the sample data to be loaded automatically
     contexts: dev, faker

backend/src/main/resources/config/application.yml

@@ -104,6 +104,7 @@ spring:
       repositories:
         bootstrap-mode: deferred
   jpa:
+    database-platform: org.hibernate.dialect.PostgreSQLDialect
     open-in-view: false
     properties:
       hibernate.jdbc.time_zone: UTC
@@ -173,4 +174,3 @@ giskard:
   regressionThreshold: 0.15
   licensePublicKey: c947f66224d465b50004c327fc831cff672fc07b540b0613d6f661d0e72d455d
 
-

backend/src/main/resources/config/liquibase/changelog/20230503092614_changelog.xml

@@ -14,12 +14,6 @@
             <column name="column_type" type="varchar(255)"/>
         </addColumn>
     </changeSet>
-    <changeSet author="kevinmessiaen (generated)" id="1683098778368-4" failOnError="false">
-        <dropForeignKeyConstraint baseTableName="TEST_INPUT" constraintName="FK5IRXPPO64AWH117G7Y2KN9IYD"/>
-    </changeSet>
-    <changeSet author="kevinmessiaen (generated)" id="1683098778368-5" failOnError="false">
-        <dropForeignKeyConstraint baseTableName="SLICES" constraintName="FKOKT2OV2HCS1K02321MCXJFVHH"/>
-    </changeSet>
     <changeSet author="kevinmessiaen (generated)" id="1683098778368-6">
         <dropTable tableName="SLICES"/>
     </changeSet>

backend/src/main/resources/logback-spring.xml

@@ -12,8 +12,6 @@
     <logger name="com.ryantenney" level="WARN"/>
     <logger name="com.sun" level="WARN"/>
     <logger name="com.zaxxer" level="WARN"/>
-    <logger name="io.undertow" level="WARN"/>
-    <logger name="io.undertow.websockets.jsr" level="ERROR"/>
     <logger name="org.apache" level="WARN"/>
     <logger name="org.apache.catalina.startup.DigesterFactory" level="OFF"/>
     <logger name="org.bson" level="WARN"/>

backend/src/test/java/ai/giskard/GiskardTestApp.java

@@ -0,0 +1,11 @@
+package ai.giskard;
+
+import org.springframework.boot.SpringApplication;
+
+public class GiskardTestApp {
+    public static void main(String[] args) {
+        SpringApplication.from(GiskardApp::main)
+            .with(RunContainersConfiguration.class)
+            .run(args);
+    }
+}

backend/src/test/java/ai/giskard/RunContainersConfiguration.java

@@ -0,0 +1,52 @@
+package ai.giskard;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.devtools.restart.RestartScope;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
+import org.springframework.context.annotation.Bean;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+@TestConfiguration(proxyBeanMethods = false)
+public class RunContainersConfiguration {
+    @Value("${spring.datasource.username:giskard}")
+    private String pgUsername;
+    @Value("${spring.datasource.password:giskard}")
+    private String pgPassowrd;
+
+
+    @Bean
+    @ServiceConnection
+    @RestartScope
+    PostgreSQLContainer<?> postgreSQLContainer() {
+
+
+        List<String> portBindings = new ArrayList<>();
+        portBindings.add("15432:5432"); // hostPort:containerPort
+
+        PostgreSQLContainer<?> pgContainer = new PostgreSQLContainer<>("postgres:13")
+            .withReuse(true)
+            .withUsername(pgUsername)
+            .withPassword(pgPassowrd)
+            .withDatabaseName("giskard")
+            .withStartupTimeout(java.time.Duration.ofSeconds(60))
+            .withFileSystemBind(resolveFromHome(Paths.get("giskard-home", "database")), "/var/lib/postgresql/data", BindMode.READ_WRITE);
+
+        pgContainer.setPortBindings(portBindings);
+        pgContainer.setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*database system is ready to accept connections.*"));
+
+        return pgContainer;
+    }
+
+    private String resolveFromHome(Path path) {
+        return Paths.get(System.getProperty("user.home")).resolve(path).toString();
+    }
+
+}