Skip to content

Commit 3940f40

Browse files
Dilyan MarinovDeltaMichael
Dilyan Marinov
authored and
DeltaMichael
committed
vdk-structlog: put vdk init logs config behind flag
Why? Configuring the logs in the vdk_initialize hook causes commands like vdk server and vdk info to have the same verbose logging format as data job logs. What? - Put the log configuration for vdk_initialize behind a config option and disable it by default for local runs and enable it by default for cloud runs. Configuring the logging format in vdk_initialize might still be useful for cloud runs and we don't care about other vdk commands output in cloud environments - Move config classes and functions into separate file How was this tested Manually CI/CD What kind of change is this? Feature/non-breaking Signed-off-by: Dilyan Marinov <mdilyan@vmware.com>
1 parent ebdd04f commit 3940f40

File tree

4 files changed

+206
-188
lines changed

4 files changed

+206
-188
lines changed

‎projects/vdk-plugins/vdk-structlog/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@ pip install vdk-structlog
2323

2424
| Name | Description | Example Value | Possible Values |
2525
|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
26-
| use_structlog | Use the structlog logging config instead of using the one in vdk-core | "True" | "True", "False" |
26+
| use_structlog | Use the structlog logging config instead of using the one in vdk-core (True by default). | "True" | "True", "False" |
2727
| structlog_metadata | Configure the metadata that will be output along with the log message | "timestamp, level, logger_name, file_name, vdk_job_name | Any combination of the following: "timestamp, level, logger_name, file_name, line_number, function_name, vdk_job_name, vdk_step_name, vdk_step_type". Can be expanded by extra params and bound key-value pairs. See the bound logger examples for more information |
2828
| structlog_format | Configure the logging output format. Available formats: json, console, ltsv | "console" | "console", "json", "ltsv" |
2929
| structlog_console_log_pattern | Custom format string for console logging, applied only when`logging_format` is 'console'. Overrides `logging_metadata`. Note: For config.ini, %-signs should be escaped by doubling, e.g. %(asctime)s should become %%(asctime)s | "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" | Any valid Python logging format string |
3030
| structlog_config_preset | Choose a configuration preset. Any config options set together with the preset will override the preset options. Available presets: LOCAL, CLOUD. | "CLOUD" | "console", "json", "ltsv" |
31+
| structlog_format_init_logs | Set to True to apply structlog formatting options to vdk initialization logs | "True" | "True", "False" |
3132
| log_level_module | Configure the log level of different Python modules separately | "a.b.c=INFO;foo.bar=ERROR" | Semicolon-separated list of pairs of Python module paths and log level labels |
3233
| syslog_host | Syslog host to which logs are emitted | "syslog.vmware.com" | Any valid host name |
3334
| syslog_port | Syslog port used to emit logs | 514 | Any valid port number |

‎projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
STRUCTLOG_LOGGING_FORMAT_KEY = "structlog_format"
99
STRUCTLOG_CONSOLE_LOG_PATTERN = "structlog_console_custom_format"
1010
STRUCTLOG_CONFIG_PRESET = "structlog_config_preset"
11+
STRUCTLOG_FORMAT_INIT_LOGS = "structlog_format_init_logs"
1112

1213
STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES = ["console", "json", "ltsv"]
1314
STRUCTLOG_LOGGING_FORMAT_DEFAULT = "console"

‎projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_config.py‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
# Copyright 2021-2024 VMware, Inc.
2+
# SPDX-License-Identifier: Apache-2.0
3+
from vdk.internal.builtin_plugins.config import vdk_config
4+
from vdk.internal.core.config import Configuration
5+
from vdk.internal.core.config import ConfigurationBuilder
6+
from vdk.plugin.structlog.constants import *
7+
8+
9+
class StructlogConfig:
10+
def __init__(self, configuration: Configuration):
11+
presets = {
12+
"LOCAL": {
13+
STRUCTLOG_USE_STRUCTLOG: configuration.get_value(
14+
STRUCTLOG_USE_STRUCTLOG
15+
),
16+
STRUCTLOG_LOGGING_METADATA_KEY: ",".join(
17+
list(JSON_STRUCTLOG_LOGGING_METADATA_DEFAULT.keys())
18+
),
19+
STRUCTLOG_LOGGING_FORMAT_KEY: STRUCTLOG_LOGGING_FORMAT_DEFAULT,
20+
STRUCTLOG_CONSOLE_LOG_PATTERN: "",
21+
STRUCTLOG_CONFIG_PRESET: configuration.get_value(
22+
STRUCTLOG_CONFIG_PRESET
23+
),
24+
SYSLOG_HOST_KEY: DEFAULT_SYSLOG_HOST,
25+
SYSLOG_PORT_KEY: DEFAULT_SYSLOG_PORT,
26+
SYSLOG_PROTOCOL_KEY: DEFAULT_SYSLOG_PROTOCOL,
27+
SYSLOG_ENABLED_KEY: DEFAULT_SYSLOG_ENABLED,
28+
vdk_config.LOG_LEVEL_VDK.lower(): configuration.get_value(
29+
vdk_config.LOG_LEVEL_VDK.lower()
30+
),
31+
vdk_config.LOG_LEVEL_MODULE.lower(): configuration.get_value(
32+
vdk_config.LOG_LEVEL_MODULE.lower()
33+
),
34+
STRUCTLOG_FORMAT_INIT_LOGS: False,
35+
},
36+
"CLOUD": {
37+
STRUCTLOG_USE_STRUCTLOG: configuration.get_value(
38+
STRUCTLOG_USE_STRUCTLOG
39+
),
40+
STRUCTLOG_LOGGING_METADATA_KEY: "",
41+
STRUCTLOG_LOGGING_FORMAT_KEY: STRUCTLOG_LOGGING_FORMAT_DEFAULT,
42+
STRUCTLOG_CONSOLE_LOG_PATTERN: DETAILED_LOGGING_FORMAT,
43+
STRUCTLOG_CONFIG_PRESET: configuration.get_value(
44+
STRUCTLOG_CONFIG_PRESET
45+
),
46+
SYSLOG_HOST_KEY: DEFAULT_SYSLOG_HOST,
47+
SYSLOG_PORT_KEY: DEFAULT_SYSLOG_PORT,
48+
SYSLOG_PROTOCOL_KEY: DEFAULT_SYSLOG_PROTOCOL,
49+
SYSLOG_ENABLED_KEY: DEFAULT_SYSLOG_ENABLED,
50+
vdk_config.LOG_LEVEL_VDK.lower(): configuration.get_value(
51+
vdk_config.LOG_LEVEL_VDK.lower()
52+
),
53+
vdk_config.LOG_LEVEL_MODULE.lower(): configuration.get_value(
54+
vdk_config.LOG_LEVEL_MODULE.lower()
55+
),
56+
STRUCTLOG_FORMAT_INIT_LOGS: True,
57+
},
58+
}
59+
60+
self._config = presets[configuration.get_value(STRUCTLOG_CONFIG_PRESET)]
61+
62+
for key in configuration.list_config_keys():
63+
if not configuration.is_default(key):
64+
self._config[key] = configuration.get_value(key)
65+
66+
def get_use_structlog(self) -> bool:
67+
return self._config[STRUCTLOG_USE_STRUCTLOG]
68+
69+
def get_structlog_logging_metadata(self) -> str:
70+
return self._config[STRUCTLOG_LOGGING_METADATA_KEY]
71+
72+
def get_structlog_logging_format(self) -> str:
73+
return self._config[STRUCTLOG_LOGGING_FORMAT_KEY]
74+
75+
def get_structlog_console_log_pattern(self) -> str:
76+
return self._config[STRUCTLOG_CONSOLE_LOG_PATTERN]
77+
78+
def get_structlog_config_preset(self) -> str:
79+
return self._config[STRUCTLOG_CONFIG_PRESET]
80+
81+
def get_syslog_host(self) -> str:
82+
return self._config[SYSLOG_HOST_KEY]
83+
84+
def get_syslog_port(self) -> int:
85+
return self._config[SYSLOG_PORT_KEY]
86+
87+
def get_syslog_protocol(self) -> str:
88+
return self._config[SYSLOG_PROTOCOL_KEY]
89+
90+
def get_syslog_enabled(self) -> bool:
91+
return self._config[SYSLOG_ENABLED_KEY]
92+
93+
def get_log_level_vdk(self) -> str:
94+
return self._config[vdk_config.LOG_LEVEL_VDK.lower()]
95+
96+
def get_log_level_module(self) -> str:
97+
return self._config[vdk_config.LOG_LEVEL_MODULE.lower()]
98+
99+
def get_format_init_logs(self) -> str:
100+
return self._config[STRUCTLOG_FORMAT_INIT_LOGS]
101+
102+
103+
def add_definitions(config_builder: ConfigurationBuilder):
104+
config_builder.add(
105+
key=STRUCTLOG_LOGGING_METADATA_KEY,
106+
default_value=",".join(list(JSON_STRUCTLOG_LOGGING_METADATA_DEFAULT.keys())),
107+
description=(
108+
f"Possible values: {STRUCTLOG_LOGGING_METADATA_ALL_KEYS}"
109+
"User-defined key-value pairs added to the logger's context will be displayed after the metadata, "
110+
"but before the message"
111+
"Keys for user-defined key-value pairs have to be added in this config option for the values to be "
112+
"displayed in the metadata"
113+
),
114+
)
115+
116+
config_builder.add(
117+
key=STRUCTLOG_CONSOLE_LOG_PATTERN,
118+
default_value="",
119+
description="Custom format string for console logging. Leave empty for default format.",
120+
)
121+
122+
config_builder.add(
123+
key=STRUCTLOG_LOGGING_FORMAT_KEY,
124+
default_value=STRUCTLOG_LOGGING_FORMAT_DEFAULT,
125+
description=(
126+
f"Controls the logging output format. Possible values: {STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES}"
127+
),
128+
)
129+
130+
config_builder.add(
131+
key=SYSLOG_HOST_KEY,
132+
default_value=DEFAULT_SYSLOG_HOST,
133+
description="Hostname of the Syslog server.",
134+
)
135+
136+
config_builder.add(
137+
key=SYSLOG_PORT_KEY,
138+
default_value=DEFAULT_SYSLOG_PORT,
139+
description="Port of the Syslog server.",
140+
)
141+
142+
config_builder.add(
143+
key=SYSLOG_PROTOCOL_KEY,
144+
default_value=DEFAULT_SYSLOG_PROTOCOL,
145+
description="Syslog protocol (UDP or TCP).",
146+
)
147+
148+
config_builder.add(
149+
key=SYSLOG_ENABLED_KEY,
150+
default_value=DEFAULT_SYSLOG_ENABLED,
151+
description="Enable Syslog logging (True or False).",
152+
)
153+
154+
config_builder.add(
155+
key=STRUCTLOG_CONFIG_PRESET,
156+
default_value="LOCAL",
157+
description="Choose configuration preset. Any config options set together with the preset will override "
158+
"the preset options. Available presets: LOCAL, CLOUD",
159+
)
160+
161+
config_builder.add(
162+
key=STRUCTLOG_USE_STRUCTLOG,
163+
default_value=True,
164+
description="Use the structlog logging config instead of using the one in vdk-core",
165+
)
166+
167+
config_builder.add(
168+
key=STRUCTLOG_FORMAT_INIT_LOGS,
169+
default_value=False,
170+
description="Set to True to apply structlog formatting options to the vdk initialization logs",
171+
)

0 commit comments

Comments
 (0)