diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 13c46141b4fb..3f163fb0ddb1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -160,7 +160,8 @@ Running System Tests can be downloaded directly from the developer's console by clicking "Generate new JSON key". See private key `docs `__ - for more details. + for more details. In order for Logging system tests to work, the Service Account + will also have to be made a project Owner. This can be changed under "IAM & Admin". - Examples of these can be found in ``system_tests/local_test_setup.sample``. We recommend copying this to ``system_tests/local_test_setup``, editing the diff --git a/docs/index.rst b/docs/index.rst index df0aa0ea9980..4ee9edc56f83 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -109,6 +109,7 @@ logging-entries logging-metric logging-sink + logging-handlers .. toctree:: :maxdepth: 0 diff --git a/docs/logging-handlers.rst b/docs/logging-handlers.rst new file mode 100644 index 000000000000..c1c0cf59703b --- /dev/null +++ b/docs/logging-handlers.rst @@ -0,0 +1,7 @@ +Python Logging Module Handler +============================== + +.. automodule:: gcloud.logging.handlers + :members: + :show-inheritance: + diff --git a/docs/logging-usage.rst b/docs/logging-usage.rst index 9b23e2edd8ab..21769d871713 100644 --- a/docs/logging-usage.rst +++ b/docs/logging-usage.rst @@ -314,3 +314,45 @@ Delete a sink: >>> sink.delete() # API call >>> sink.exists() # API call False + +Integration with Python logging module +--------------------------------------------- + + +It's possible to tie the Python :mod:`logging` module directly into Google Cloud Logging. To use it, +create a :class:`CloudLoggingHandler ` instance from your +Logging client. + +.. doctest:: + + >>> import logging + >>> import gcloud.logging # Don't conflict with standard logging + >>> from gcloud.logging.handlers import CloudLoggingHandler + >>> client = gcloud.logging.Client() + >>> handler = CloudLoggingHandler(client) + >>> cloud_logger = logging.getLogger('cloudLogger') + >>> cloud_logger.setLevel(logging.INFO) # defaults to WARN + >>> cloud_logger.addHandler(handler) + >>> cloud_logger.error('bad news') # API call + +.. note:: + + This handler currently only supports a synchronous API call, which means each logging statement + that uses this handler will require an API call. + +It is also possible to attach the handler to the root Python logger, so that for example a plain +`logging.warn` call would be sent to Cloud Logging, as well as any other loggers created. However, +you must avoid infinite recursion from the logging calls the client itself makes. A helper +method :meth:`setup_logging ` is provided to configure +this automatically: + +.. doctest:: + + >>> import logging + >>> import gcloud.logging # Don't conflict with standard logging + >>> from gcloud.logging.handlers import CloudLoggingHandler, setup_logging + >>> client = gcloud.logging.Client() + >>> handler = CloudLoggingHandler(client) + >>> logging.getLogger().setLevel(logging.INFO) # defaults to WARN + >>> setup_logging(handler) + >>> logging.error('bad news') # API call diff --git a/gcloud/logging/handlers.py b/gcloud/logging/handlers.py new file mode 100644 index 000000000000..9165b091f5db --- /dev/null +++ b/gcloud/logging/handlers.py @@ -0,0 +1,110 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python :mod:`logging` handlers for Google Cloud Logging.""" + +import logging + +EXCLUDE_LOGGER_DEFAULTS = ( + 'gcloud', + 'oauth2client.client' +) + + +class CloudLoggingHandler(logging.StreamHandler, object): + """Python standard logging handler to log messages to the Google Cloud + Logging API. + + This handler can be used to route Python standard logging messages to + Google Cloud logging. + + Note that this handler currently only supports a synchronous API call, + which means each logging statement that uses this handler will require + an API call. + + :type client: :class:`gcloud.logging.client` + :param client: the authenticated gcloud logging client for this handler + to use + + Example: + + .. doctest:: + + import gcloud.logging + from gcloud.logging.handlers import CloudLoggingHandler + + client = gcloud.logging.Client() + handler = CloudLoggingHandler(client) + + cloud_logger = logging.getLogger('cloudLogger') + cloud_logger.setLevel(logging.INFO) + cloud_logger.addHandler(handler) + + cloud.logger.error("bad news") # API call + + """ + + def __init__(self, client): + super(CloudLoggingHandler, self).__init__() + self.client = client + + def emit(self, record): + """ + Overrides the default emit behavior of StreamHandler. + + See: https://docs.python.org/2/library/logging.html#handler-objects + """ + message = super(CloudLoggingHandler, self).format(record) + logger = self.client.logger(record.name) + logger.log_struct({"message": message}, + severity=record.levelname) + + +def setup_logging(handler, excluded_loggers=EXCLUDE_LOGGER_DEFAULTS): + """Helper function to attach the CloudLoggingAPI handler to the Python + root logger, while excluding loggers this library itself uses to avoid + infinite recursion + + :type handler: :class:`logging.handler` + :param handler: the handler to attach to the global handler + + :type excluded_loggers: tuple + :param excluded_loggers: The loggers to not attach the handler to. This + will always include the loggers in the path of + the logging client itself. + + Example: + + .. doctest:: + + import logging + import gcloud.logging + from gcloud.logging.handlers import CloudLoggingAPIHandler + + client = gcloud.logging.Client() + handler = CloudLoggingHandler(client) + setup_logging(handler) + logging.getLogger().setLevel(logging.DEBUG) + + logging.error("bad news") # API call + + """ + all_excluded_loggers = set(excluded_loggers + EXCLUDE_LOGGER_DEFAULTS) + logger = logging.getLogger() + logger.addHandler(handler) + logger.addHandler(logging.StreamHandler()) + for logger_name in all_excluded_loggers: + logger = logging.getLogger(logger_name) + logger.propagate = False + logger.addHandler(logging.StreamHandler()) diff --git a/gcloud/logging/test_handlers.py b/gcloud/logging/test_handlers.py new file mode 100644 index 000000000000..73095e2c50c8 --- /dev/null +++ b/gcloud/logging/test_handlers.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import unittest2 + + +class TestCloudLoggingHandler(unittest2.TestCase): + + PROJECT = 'PROJECT' + + def _getTargetClass(self): + from gcloud.logging.handlers import CloudLoggingHandler + return CloudLoggingHandler + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor(self): + client = _Client(self.PROJECT) + handler = self._makeOne(client) + self.assertEqual(handler.client, client) + + def test_emit(self): + client = _Client(self.PROJECT) + handler = self._makeOne(client) + LOGNAME = 'loggername' + MESSAGE = 'hello world' + record = _Record(LOGNAME, logging.INFO, MESSAGE) + handler.emit(record) + self.assertEqual(client.logger(LOGNAME).log_struct_called_with, + ({'message': MESSAGE}, logging.INFO)) + + +class TestSetupLogging(unittest2.TestCase): + + def _callFUT(self, handler, excludes=None): + from gcloud.logging.handlers import setup_logging + if excludes: + return setup_logging(handler, excluded_loggers=excludes) + else: + return setup_logging(handler) + + def test_setup_logging(self): + handler = _Handler(logging.INFO) + self._callFUT(handler) + + root_handlers = logging.getLogger().handlers + self.assertIn(handler, root_handlers) + + def test_setup_logging_excludes(self): + INCLUDED_LOGGER_NAME = 'includeme' + EXCLUDED_LOGGER_NAME = 'excludeme' + + handler = _Handler(logging.INFO) + self._callFUT(handler, (EXCLUDED_LOGGER_NAME,)) + + included_logger = logging.getLogger(INCLUDED_LOGGER_NAME) + self.assertTrue(included_logger.propagate) + + excluded_logger = logging.getLogger(EXCLUDED_LOGGER_NAME) + self.assertNotIn(handler, excluded_logger.handlers) + self.assertFalse(excluded_logger.propagate) + + def setUp(self): + self._handlers_cache = logging.getLogger().handlers[:] + + def tearDown(self): + # cleanup handlers + logging.getLogger().handlers = self._handlers_cache[:] + + +class _Handler(object): + + def __init__(self, level): + self.level = level + + def acquire(self): + pass # pragma: NO COVER + + def release(self): + pass # pragma: NO COVER + + +class _Logger(object): + + def log_struct(self, message, severity=None): + self.log_struct_called_with = (message, severity) + + +class _Client(object): + + def __init__(self, project): + self.project = project + self.logger_ = _Logger() + + def logger(self, _): # pylint: disable=unused-argument + return self.logger_ + + +class _Record(object): + + def __init__(self, name, level, message): + self.name = name + self.levelname = level + self.message = message + self.exc_info = None + self.exc_text = None + self.stack_info = None + + def getMessage(self): + return self.message diff --git a/system_tests/logging_.py b/system_tests/logging_.py index db29f854d4d0..6b47ef56aecc 100644 --- a/system_tests/logging_.py +++ b/system_tests/logging_.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import time import unittest2 from gcloud import _helpers from gcloud.environment_vars import TESTS_PROJECT -from gcloud import logging +import gcloud.logging +import gcloud.logging.handlers from system_test_utils import unique_resource_id @@ -44,13 +46,14 @@ class Config(object): def setUpModule(): _helpers.PROJECT = TESTS_PROJECT - Config.CLIENT = logging.Client() + Config.CLIENT = gcloud.logging.Client() class TestLogging(unittest2.TestCase): def setUp(self): self.to_delete = [] + self._handlers_cache = logging.getLogger().handlers[:] def tearDown(self): from gcloud.exceptions import NotFound @@ -65,6 +68,7 @@ def tearDown(self): time.sleep(backoff_intervals.pop(0)) else: raise + logging.getLogger().handlers = self._handlers_cache[:] @staticmethod def _logger_name(): @@ -75,7 +79,7 @@ def test_log_text(self): logger = Config.CLIENT.logger(self._logger_name()) self.to_delete.append(logger) logger.log_text(TEXT_PAYLOAD) - time.sleep(2) + time.sleep(5) entries, _ = logger.list_entries() self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, TEXT_PAYLOAD) @@ -96,7 +100,7 @@ def test_log_text_w_metadata(self): self.to_delete.append(logger) logger.log_text(TEXT_PAYLOAD, insert_id=INSERT_ID, severity=SEVERITY, http_request=REQUEST) - time.sleep(2) + time.sleep(5) entries, _ = logger.list_entries() self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, TEXT_PAYLOAD) @@ -115,11 +119,40 @@ def test_log_struct(self): logger = Config.CLIENT.logger(self._logger_name()) self.to_delete.append(logger) logger.log_struct(JSON_PAYLOAD) - time.sleep(2) + time.sleep(5) entries, _ = logger.list_entries() self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, JSON_PAYLOAD) + def test_log_handler(self): + LOG_MESSAGE = 'It was the best of times.' + # only create the logger to delete, hidden otherwise + logger = Config.CLIENT.logger(self._logger_name()) + self.to_delete.append(logger) + + handler = gcloud.logging.handlers.CloudLoggingHandler(Config.CLIENT) + cloud_logger = logging.getLogger(self._logger_name()) + cloud_logger.addHandler(handler) + cloud_logger.warn(LOG_MESSAGE) + time.sleep(5) + entries, _ = logger.list_entries() + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].payload, {'message': LOG_MESSAGE}) + + def test_log_root_handler(self): + LOG_MESSAGE = 'It was the best of times.' + # only create the logger to delete, hidden otherwise + logger = Config.CLIENT.logger("root") + self.to_delete.append(logger) + + handler = gcloud.logging.handlers.CloudLoggingHandler(Config.CLIENT) + gcloud.logging.handlers.setup_logging(handler) + logging.warn(LOG_MESSAGE) + time.sleep(5) + entries, _ = logger.list_entries() + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].payload, {'message': LOG_MESSAGE}) + def test_log_struct_w_metadata(self): JSON_PAYLOAD = { 'message': 'System test: test_log_struct', @@ -139,7 +172,7 @@ def test_log_struct_w_metadata(self): self.to_delete.append(logger) logger.log_struct(JSON_PAYLOAD, insert_id=INSERT_ID, severity=SEVERITY, http_request=REQUEST) - time.sleep(2) + time.sleep(5) entries, _ = logger.list_entries() self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, JSON_PAYLOAD)