Add Helm job for OpenSearch index creation #36
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| { | ||
| "mappings": { | ||
| "properties": { | ||
| "object_ref": { "type": "keyword" }, | ||
| "object_type": { "type": "keyword" }, | ||
| "object_id": { "type": "keyword" }, | ||
| "parent_refs": { "type": "keyword" }, | ||
| "sort_name": { "type": "keyword" }, | ||
| "name_and_aliases": { "type": "search_as_you_type" }, | ||
| "tags": { "type": "keyword" }, | ||
| "public": { "type": "boolean" }, | ||
| "access_check_query": { "type": "keyword" }, | ||
| "history_check_query": { "type": "keyword" }, | ||
| "latest": { "type": "boolean" }, | ||
| "created_at": { "type": "date" }, | ||
| "created_by": { "type": "keyword" }, | ||
| "created_by_principals": { "type": "keyword" }, | ||
| "created_by_emails": { "type": "keyword" }, | ||
| "updated_at": { "type": "date" }, | ||
| "updated_by": { "type": "keyword" }, | ||
| "updated_by_principals": { "type": "keyword" }, | ||
| "updated_by_emails": { "type": "keyword" }, | ||
| "deleted_at": { "type": "date" }, | ||
| "deleted_by": { "type": "keyword" }, | ||
| "deleted_by_principals": { "type": "keyword" }, | ||
| "deleted_by_emails": { "type": "keyword" }, | ||
| "data": { | ||
| "type": "flat_object" | ||
| }, | ||
| "fulltext": { | ||
| "type": "match_only_text" | ||
| }, | ||
| "contacts": { | ||
| "type": "nested", | ||
| "properties": { | ||
| "lfx_principal": { "type": "search_as_you_type" }, | ||
| "name": { "type": "search_as_you_type" }, | ||
| "emails": { "type": "search_as_you_type" }, | ||
| "bot": { "type": "boolean" }, | ||
| "profile": { "type": "flat_object" } | ||
| } | ||
| }, | ||
| "v1_data": { | ||
| "type": "flat_object" | ||
| } | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # Copyright The Linux Foundation and each contributor to LFX. | ||
| # SPDX-License-Identifier: MIT | ||
| {{- if .Values.opensearch.indexingJob.enabled }} | ||
| --- | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: {{ .Release.Name }}-opensearch-index-config | ||
| namespace: {{ .Release.Namespace }} | ||
| data: | ||
| resources-index.json: | | ||
| {{ .Files.Get "files/opensearch-resources-index.json" | nindent 4 }} | ||
| {{- end }} | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| # Copyright The Linux Foundation and each contributor to LFX. | ||
| # SPDX-License-Identifier: MIT | ||
| {{- if .Values.opensearch.indexingJob.enabled }} | ||
| {{- $job := .Values.opensearch.indexingJob }} | ||
| --- | ||
| apiVersion: batch/v1 | ||
| kind: Job | ||
| metadata: | ||
| name: {{ .Release.Name }}-opensearch-index-setup | ||
| namespace: {{ .Release.Namespace }} | ||
| spec: | ||
| ttlSecondsAfterFinished: {{ $job.ttlSecondsAfterFinished }} | ||
| backoffLimit: {{ $job.backoffLimit }} | ||
| {{- if $job.activeDeadlineSeconds }} | ||
| activeDeadlineSeconds: {{ $job.activeDeadlineSeconds }} | ||
| {{- end }} | ||
| template: | ||
| spec: | ||
| restartPolicy: {{ $job.restartPolicy }} | ||
| volumes: | ||
| - name: index-config | ||
| configMap: | ||
| name: {{ .Release.Name }}-opensearch-index-config | ||
| containers: | ||
| - name: curl | ||
| image: {{ $job.image.repository }}:{{ $job.image.tag }} | ||
| imagePullPolicy: {{ $job.image.pullPolicy }} | ||
| {{- with $job.resources }} | ||
| resources: | ||
| {{- toYaml . | nindent 12 }} | ||
| {{- end }} | ||
| {{- if .Values.opensearch.auth.enabled }} | ||
| env: | ||
| - name: OPENSEARCH_USERNAME | ||
| {{- if .Values.opensearch.auth.existingSecret }} | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: {{ .Values.opensearch.auth.existingSecret }} | ||
| key: username | ||
| {{- else }} | ||
| value: {{ .Values.opensearch.auth.username | quote }} | ||
| {{- end }} | ||
| - name: OPENSEARCH_PASSWORD | ||
| {{- if .Values.opensearch.auth.existingSecret }} | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: {{ .Values.opensearch.auth.existingSecret }} | ||
| key: password | ||
| {{- else }} | ||
| value: {{ .Values.opensearch.auth.password | quote }} | ||
| {{- end }} | ||
| {{- end }} | ||
| volumeMounts: | ||
| - name: index-config | ||
| mountPath: /config | ||
| readOnly: true | ||
| command: | ||
| - sh | ||
| - -c | ||
| - | | ||
| set -e | ||
| OPENSEARCH_URL="{{ .Values.opensearch.url | trimSuffix "/" }}" | ||
| INDEX_NAME="{{ .Values.opensearch.index }}" | ||
| {{- if .Values.opensearch.auth.enabled }} | ||
| AUTH_OPTS="-u ${OPENSEARCH_USERNAME}:${OPENSEARCH_PASSWORD}" | ||
| {{- end }} | ||
|
|
||
| # Check if index already exists | ||
| HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" {{ if .Values.opensearch.auth.enabled }}${AUTH_OPTS} {{ end }}-X HEAD "${OPENSEARCH_URL}/${INDEX_NAME}") | ||
|
|
||
| if [ "$HTTP_CODE" = "200" ]; then | ||
| echo "Index '${INDEX_NAME}' already exists, skipping creation" | ||
| exit 0 | ||
| elif [ "$HTTP_CODE" = "404" ]; then | ||
| echo "Index '${INDEX_NAME}' does not exist, creating..." | ||
| curl -f {{ if .Values.opensearch.auth.enabled }}${AUTH_OPTS} {{ end }}-X PUT "${OPENSEARCH_URL}/${INDEX_NAME}" \ | ||
| -H 'Content-Type: application/json' \ | ||
| -d @/config/resources-index.json | ||
| echo "Index '${INDEX_NAME}' created successfully" | ||
| else | ||
| echo "Unexpected response checking index: HTTP ${HTTP_CODE}" | ||
| exit 1 | ||
| fi | ||
|
Comment on lines
+68
to
+83
There was a problem hiding this comment. 🧩 Analysis chain🏁 Script executed: cd charts/lfx-v2-indexer-service && find . -type f -name "*.yaml" -o -name "*.yml" | head -20Repository: linuxfoundation/lfx-v2-indexer-service Length of output: 193 🏁 Script executed: cat -n charts/lfx-v2-indexer-service/templates/job.yaml | sed -n '60,90p'Repository: linuxfoundation/lfx-v2-indexer-service Length of output: 1524 🏁 Script executed: cat charts/lfx-v2-indexer-service/values.yamlRepository: linuxfoundation/lfx-v2-indexer-service Length of output: 4563 🏁 Script executed: #!/bin/bash
# Check if the image tag exists on Docker Hub
echo "Checking Docker Hub for curlimages/curl:8.11.1..."
DOCKER_RESPONSE=$(curl -s "https://hub.docker.com/v2/repositories/curlimages/curl/tags/8.11.1/")
echo "$DOCKER_RESPONSE" | jq -r '.name // "NOT_FOUND"'
# Also check available tags to verify repository access
echo -e "\nRecent available tags (first 5):"
curl -s "https://hub.docker.com/v2/repositories/curlimages/curl/tags/?page_size=5" | jq -r '.results[].name'Repository: linuxfoundation/lfx-v2-indexer-service Length of output: 276 🌐 Web query:
💡 Result: curl 8.11.1 had multiple security advisories fixed in curl 8.12.0 (released Feb 5, 2025). Summary of CVEs affecting 8.11.1:
Recommended action: upgrade curl/libcurl to 8.12.0 (or later) or apply the vendor patches; if you cannot upgrade immediately follow the per-advisory mitigations (avoid netrc+redirects, avoid CURLOPT_ACCEPT_ENCODING with ancient zlib, disable eventfd) until patched. [1][2][3] Sources: Upgrade curl image to version 8.12.0 or later to address known security advisories. The index creation logic is well-structured with proper idempotency checks (HEAD before PUT) and appropriate error handling for different HTTP response codes. However, curlimages/curl:8.11.1 has three low-severity security advisories that were fixed in 8.12.0 (released February 5, 2025):
Update the image tag in values.yaml from 🤖 Prompt for AI Agents |
||
| {{- end }} | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This index json content doesn't quite match what we should be actually using, which is https://github.com/linuxfoundation/lfx-architecture-scratch/blob/main/2024-12%20ReBAC%20Demo/opensearch-resources-index.json. But on that note, could we have this json content in a file instead and then have it read into the k8 template? I think that would be better.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the feedback! I realized I hadn't pulled the repo for a while and was using an outdated index.
I've moved the index JSON to it's own file and have it being included now with
.Files.Get. I also update the job to not attempt to create the index if it already exists.