Skip to content

[7.x] Add xpack core and security to INTEG TEST (#77632) #79326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jkakavas wants to merge 2 commits into from
Closed

[7.x] Add xpack core and security to INTEG TEST (#77632) #79326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4355afe
Add xpack core and security to INTEG TEST (#77632)
jkakavas Sep 21, 2021
f27cfd6
revert unecessary changes
jkakavas Oct 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions distribution/archives/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
return copySpec {
into("elasticsearch-${version}") {
into('lib') {
with libFiles(oss)
with libFiles
}
into('config') {
dirMode 0750
Expand Down Expand Up @@ -65,7 +65,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
distribution_archives {
integTestZip {
content {
archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
archiveFiles(integTestModulesFiles, 'zip', null, 'x64', true, false)
}
}

Expand Down
34 changes: 18 additions & 16 deletions distribution/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ tasks.withType(NoticeTask).configureEach {
String ossOutputs = 'build/outputs/oss'
String defaultOutputs = 'build/outputs/default'
String systemdOutputs = 'build/outputs/systemd'
String transportOutputs = 'build/outputs/transport-only'
String integTestOutputs = 'build/outputs/integ-test-only'
String externalTestOutputs = 'build/outputs/external-test'

def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) {
Expand All @@ -106,8 +106,8 @@ def processExternalTestOutputsTaskProvider = tasks.register("processExternalTest

// Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
// All transport modules are included so that they may be randomized for testing
def processTransportOutputsTaskProvider = tasks.register("processTransportOutputs", Sync) {
into transportOutputs
def processIntegTestOutputsTaskProvider = tasks.register("processIntegTestOutputs", Sync) {
into integTestOutputs
}

def defaultModulesFiles = fileTree("${defaultOutputs}/modules") {
Expand All @@ -125,10 +125,11 @@ def systemdModuleFiles = fileTree("${systemdOutputs}/modules") {
builtBy processSystemdOutputsTaskProvider
}

def buildTransportModulesTaskProvider = tasks.register("buildTransportModules") {
dependsOn processTransportOutputsTaskProvider
outputs.dir "${transportOutputs}/modules"
def buildIntegTestModulesTaskProvider = tasks.register("buildIntegTestModules") {
dependsOn processIntegTestOutputsTaskProvider
outputs.dir "${integTestOutputs}/modules"
}

def buildExternalTestModulesTaskProvider = tasks.register("buildExternalTestModules") {
dependsOn "processExternalTestOutputs"
outputs.dir "${externalTestOutputs}/modules"
Expand Down Expand Up @@ -216,7 +217,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {

copyModule(processDefaultOutputsTaskProvider, module)
if (module.name.startsWith('transport-')) {
copyModule(processTransportOutputsTaskProvider, module)
copyModule(processIntegTestOutputsTaskProvider, module)
}

restTestExpansions['expected.modules.count'] += 1
Expand All @@ -233,6 +234,9 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
}
}
copyModule(processDefaultOutputsTaskProvider, xpackModule)
if (xpackModule.name.equals('core') || xpackModule.name.equals('security')) {
copyModule(processIntegTestOutputsTaskProvider, xpackModule)
}
}

copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd'))
Expand Down Expand Up @@ -294,7 +298,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
/*****************************************************************************
* Common files in all distributions *
*****************************************************************************/
libFiles = { testDistro ->
libFiles =
copySpec {
// delay by using closures, since they have not yet been configured, so no jar task exists yet
from(configurations.libs)
Expand All @@ -307,13 +311,11 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
into('tools/keystore-cli') {
from(configurations.libsKeystoreCli)
}
if (testDistro == false) {
into('tools/security-cli') {
from(configurations.libsSecurityCli)
}
into('tools/security-cli') {
from(configurations.libsSecurityCli)
}
}
}


modulesFiles = { platform ->
copySpec {
Expand Down Expand Up @@ -350,8 +352,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
}

transportModulesFiles = copySpec {
from buildTransportModulesTaskProvider
integTestModulesFiles = copySpec {
from buildIntegTestModulesTaskProvider
}

configFiles = { distributionType, testDistro, jdk ->
Expand Down Expand Up @@ -393,7 +395,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
// module provided bin files
with copySpec {
eachFile { it.setMode(0755) }
if(testDistro == false) {
if (testDistro == false) {
from(defaultBinFiles)
}
if (distributionType != 'zip') {
Expand Down
2 changes: 1 addition & 1 deletion distribution/packages/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk, String archit
fileMode 0644
}
into('lib') {
with libFiles(oss)
with libFiles
}
into('modules') {
with modulesFiles('linux-' + ((architecture == 'x64') ? 'x86_64' : architecture))
Expand Down
43 changes: 38 additions & 5 deletions test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,10 @@ public static boolean hasXPack() {
*/
private static RestClient adminClient;
private static Boolean hasXPack;
private static Boolean hasIlm;
private static Boolean hasRollups;
private static Boolean hasCcr;
private static Boolean hasShutdown;
private static TreeSet<Version> nodeVersions;

@Before
Expand All @@ -181,6 +185,10 @@ public void initClient() throws IOException {
assert adminClient == null;
assert clusterHosts == null;
assert hasXPack == null;
assert hasIlm == null;
assert hasRollups == null;
assert hasCcr == null;
assert hasShutdown == null;
assert nodeVersions == null;
String cluster = getTestRestCluster();
String[] stringUrls = cluster.split(",");
Expand All @@ -200,6 +208,10 @@ public void initClient() throws IOException {
adminClient = buildClient(restAdminSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()]));

hasXPack = false;
hasIlm = false;
hasRollups = false;
hasCcr = false;
hasShutdown = false;
nodeVersions = new TreeSet<>();
Map<?, ?> response = entityAsMap(adminClient.performRequest(new Request("GET", "_nodes/plugins")));
Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
Expand All @@ -208,16 +220,33 @@ public void initClient() throws IOException {
nodeVersions.add(Version.fromString(nodeInfo.get("version").toString()));
for (Object module: (List<?>) nodeInfo.get("modules")) {
Map<?, ?> moduleInfo = (Map<?, ?>) module;
if (moduleInfo.get("name").toString().startsWith("x-pack-")) {
final String moduleName = moduleInfo.get("name").toString();
if (moduleName.startsWith("x-pack")) {
hasXPack = true;
}
if (moduleName.equals("x-pack-ilm")) {
hasIlm = true;
}
if (moduleName.equals("x-pack-rollup")) {
hasRollups = true;
}
if (moduleName.equals("x-pack-ccr")) {
hasCcr = true;
}
if (moduleName.equals("x-pack-shutdown")) {
hasShutdown = true;
}
}
}
}
assert client != null;
assert adminClient != null;
assert clusterHosts != null;
assert hasXPack != null;
assert hasIlm != null;
assert hasRollups != null;
assert hasCcr != null;
assert hasShutdown != null;
assert nodeVersions != null;
}

Expand Down Expand Up @@ -382,6 +411,10 @@ public static void closeClients() throws IOException {
client = null;
adminClient = null;
hasXPack = null;
hasRollups = null;
hasCcr = null;
hasShutdown = null;
hasIlm = null;
nodeVersions = null;
}
}
Expand Down Expand Up @@ -587,7 +620,7 @@ private void wipeCluster() throws Exception {
// Cleanup rollup before deleting indices. A rollup job might have bulks in-flight,
// so we need to fully shut them down first otherwise a job might stall waiting
// for a bulk to finish against a non-existing index (and then fail tests)
if (hasXPack && false == preserveRollupJobsUponCompletion()) {
if (hasRollups && false == preserveRollupJobsUponCompletion()) {
wipeRollupJobs();
waitForPendingRollupTasks();
}
Expand Down Expand Up @@ -745,11 +778,11 @@ private void wipeCluster() throws Exception {
wipeClusterSettings();
}

if (hasXPack && false == preserveILMPoliciesUponCompletion()) {
if (hasIlm && false == preserveILMPoliciesUponCompletion()) {
deleteAllILMPolicies(preserveILMPolicyIds());
}

if (hasXPack && false == preserveAutoFollowPatternsUponCompletion()) {
if (hasCcr && false == preserveAutoFollowPatternsUponCompletion()) {
deleteAllAutoFollowPatterns();
}

Expand All @@ -763,7 +796,7 @@ private void wipeCluster() throws Exception {
*/
@SuppressWarnings("unchecked")
protected void deleteAllNodeShutdownMetadata() throws IOException {
if (hasXPack() == false || minimumNodeVersion().before(Version.V_7_15_0)) {
if (hasShutdown == false || minimumNodeVersion().before(Version.V_7_15_0)) {
// Node shutdown APIs are only present in xpack
return;
}
Expand Down