diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ea499a8..e4365e297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* **plugin:bitwarden_item**: Add file-based item cache to reduce `bw serve` API calls, preventing crashes under load. Cache is stored in `$XDG_RUNTIME_DIR` (RAM-backed tmpfs) with `/tmp` fallback. After create/edit operations, the cache is updated inline to avoid expensive full re-syncs, with a 1-second sleep as rate limit to prevent Bitwarden API errors. Convert `is_unlocked` to a property to fix it never being called. * **role:freeipa_server**: Add `--diff` support for all FreeIPA modules and add `freeipa_server:configure` tag * **role:mariadb_server**: Add `mariadb_server__cnf_wsrep_log_conflicts` and `mariadb_server__cnf_wsrep_retry_autocommit` variables * **role:mariadb_server**: Add `mariadb_server__cnf_wsrep_gtid_mode` variable to configure `wsrep_gtid_mode` for Galera @@ -69,6 +70,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* **plugin:bitwarden_item**: Fix missing `raise` in multipart error handling, `break` instead of `continue` in multi-term lookup, `folder_id` wrongly typed as `list` instead of `str` in module, notes default mismatch between documentation and code, and wrong "lookup plugin" wording in module documentation * **role:mirror**: Fix missing `0440` permissions on sudoers file * **role:login**: Rename sudoers file from `lfops_login` to `linuxfabrik` to match the kickstart configuration; remove the old file automatically * **roles**: Fix Ansible 2.19 deprecation warning for conditional results of type `int` by using `| length > 0` instead of `| length` diff --git a/plugins/lookup/bitwarden_item.py b/plugins/lookup/bitwarden_item.py index 996fc7dc7..c95e3ea7c 100644 --- a/plugins/lookup/bitwarden_item.py +++ b/plugins/lookup/bitwarden_item.py @@ -277,8 +277,6 @@ sample: 'root' ''' -import time - from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase from ansible.utils.display import Display @@ -290,9 +288,6 @@ # https://docs.ansible.com/ansible/latest/dev_guide/developing_plugins.html#developing-lookup-plugins # inspired by the lookup plugins lastpass (same topic) and redis (more modern) -SYNC_INTERVAL = 60 # seconds -SYNC_TIMESTAMP_FILE = '/tmp/lfops_bitwarden_sync_time' - class LookupModule(LookupBase): def run(self, terms, variables=None, **kwargs): @@ -302,23 +297,7 @@ def run(self, terms, variables=None, **kwargs): raise AnsibleError('Not logged into Bitwarden, or Bitwarden Vault is locked. Please run `bw login` and `bw unlock` first.') display.vvv('lfbwlp - run - bitwarden vault is unlocked') - timestamp = 0 - try: - with open(SYNC_TIMESTAMP_FILE, 'r') as f: - timestamp = float(f.read().strip()) - except (ValueError, IOError): - pass # we just sync if an error occurs - - if time.time() - timestamp >= SYNC_INTERVAL: - display.vvv('lfbwlp - run - syncing the vault') - bw.sync() - timestamp = time.time() - - try: - with open(SYNC_TIMESTAMP_FILE, 'w') as f: - f.write(str(timestamp)) - except IOError: - display.vvv('lfbwlp - run - failed to write last sync time') + bw.sync() ret = [] for term in terms: @@ -329,7 +308,7 @@ def run(self, terms, variables=None, **kwargs): hostname = term.get('hostname', None) id_ = term.get('id', None) name = term.get('name', None) - notes = term.get('notes', 'Automatically generated by Ansible.') + notes = term.get('notes', 'Generated by Ansible.') organization_id = term.get('organization_id', None) password_length = term.get('password_length', 60) password_choice = term.get('password_choice', '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') @@ -347,7 +326,7 @@ def run(self, terms, variables=None, **kwargs): result['username'] = result['login']['username'] result['password'] = result['login']['password'] ret.append(result) - break # done here, go to next term + continue # done here, go to next term else: # item not found by ID. if there is an ID given we expect it to exist raise AnsibleError('Item with id {} not found.'.format(id_)) diff --git a/plugins/module_utils/bitwarden.py b/plugins/module_utils/bitwarden.py index 6d083d9a8..cd3fba57d 100644 --- a/plugins/module_utils/bitwarden.py +++ b/plugins/module_utils/bitwarden.py @@ -6,32 +6,32 @@ from __future__ import absolute_import, division, print_function +# This module requires Python 3.8+ (secrets, f-strings with =, os.replace, json.JSONDecodeError). This should be fine since it will always run on localhost and the Ansible Controller has to be Python 3.9+ anyway + +import copy import email.encoders import email.mime.application import email.mime.multipart import email.mime.nonmultipart import email.parser -import email.utils +import email.policy import json import mimetypes import os import secrets -import urllib.parse +import tempfile +import time from urllib.error import HTTPError, URLError from ansible.module_utils.common.collections import Mapping -from ansible.module_utils.six import PY2, PY3, string_types -from ansible.module_utils.six.moves import cStringIO - -try: - import email.policy -except ImportError: - # Py2 - import email.generator +from ansible.module_utils.six import string_types from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils.urls import (ConnectionError, SSLValidationError, open_url) +from ansible.utils.display import Display + +display = Display() def prepare_multipart_no_base64(fields): @@ -120,30 +120,15 @@ def prepare_multipart_no_base64(fields): m.attach(part) - if PY3: - # Ensure headers are not split over multiple lines - # The HTTP policy also uses CRLF by default - b_data = m.as_bytes(policy=email.policy.HTTP) - else: - # Py2 - # We cannot just call ``as_string`` since it provides no way - # to specify ``maxheaderlen`` - fp = cStringIO() # cStringIO seems to be required here - # Ensure headers are not split over multiple lines - g = email.generator.Generator(fp, maxheaderlen=0) - g.flatten(m) - # ``fix_eols`` switches from ``\n`` to ``\r\n`` - b_data = email.utils.fix_eols(fp.getvalue()) + # Ensure headers are not split over multiple lines + # The HTTP policy also uses CRLF by default + b_data = m.as_bytes(policy=email.policy.HTTP) del m headers, sep, b_content = b_data.partition(b'\r\n\r\n') del b_data - if PY3: - parser = email.parser.BytesHeaderParser().parsebytes - else: - # Py2 - parser = email.parser.HeaderParser().parsestr + parser = email.parser.BytesHeaderParser().parsebytes return ( parser(headers)['content-type'], # Message converts to native strings @@ -151,6 +136,11 @@ def prepare_multipart_no_base64(fields): ) +CACHE_DIR = os.environ.get('XDG_RUNTIME_DIR', '/tmp') +CACHE_FILE = os.path.join(CACHE_DIR, 'lfops_bitwarden_cache.json') +CACHE_VERSION = 2026032701 + + class BitwardenException(Exception): pass @@ -160,6 +150,8 @@ class Bitwarden(object): def __init__(self, hostname='127.0.0.1', port=8087): self._base_url = 'http://%s:%s' % (hostname, port) + self._cache = None + self._load_cache() def _api_call(self, url_path, method='GET', body=None, body_format='json'): url = '%s/%s' % (self._base_url, url_path) @@ -173,12 +165,13 @@ def _api_call(self, url_path, method='GET', body=None, body_format='json'): try: content_type, body = prepare_multipart_no_base64(body) except (TypeError, ValueError) as e: - BitwardenException('failed to parse body as form-multipart: %s' % to_native(e)) + raise BitwardenException('failed to parse body as form-multipart: %s' % to_native(e)) headers['Content-Type'] = content_type # mostly taken from ansible.builtin.url lookup plugin try: - response = open_url(url, method=method, data=body, headers=headers) + # increased the timeout since listing all items via `list/object/items` takes forever (13s for ~2500 items) + response = open_url(url, method=method, data=body, headers=headers, timeout=60) except HTTPError as e: raise BitwardenException("Received HTTP error for %s : %s" % (url, to_native(e))) except URLError as e: @@ -199,6 +192,64 @@ def _api_call(self, url_path, method='GET', body=None, body_format='json'): return result + def _load_cache(self): + """Load the cache from disk. If missing, unreadable, or invalid, start with an empty cache. + Freshness is handled by sync(). + """ + try: + with open(CACHE_FILE, 'r') as f: + data = json.load(f) + if data.get('version') == CACHE_VERSION: + self._cache = data + item_count = len(self._cache['items']) if self._cache['items'] is not None else 0 + display.vvv('lfbw - cache loaded from %s (%d items)' % (CACHE_FILE, item_count)) + return + except (IOError, OSError, ValueError, json.decoder.JSONDecodeError): + pass + self._cache = { + 'version': CACHE_VERSION, + 'sync_timestamp': 0, + 'items': None, + 'templates': {}, + } + display.vvv('lfbw - no valid cache found, starting fresh') + + + def _save_cache(self): + """Write the cache to disk atomically. + """ + try: + fd, tmp_path = tempfile.mkstemp( + dir=os.path.dirname(CACHE_FILE), + prefix='.lfops_bw_cache_', + ) + try: + with os.fdopen(fd, 'w') as f: + json.dump(self._cache, f) + os.replace(tmp_path, CACHE_FILE) + display.vvv('lfbw - cache saved to %s' % (CACHE_FILE)) + except Exception: + os.unlink(tmp_path) + raise + except (IOError, OSError): + display.vvv('lfbw - failed to save cache to %s' % (CACHE_FILE)) + + + def _get_template(self, template_name): + """Return a template from cache, fetching from API on first use. + Templates are static API schema definitions that never change. + """ + if template_name not in self._cache['templates']: + display.vvv('lfbw - fetching template "%s" from API' % (template_name)) + result = self._api_call('object/template/%s' % (template_name)) + self._cache['templates'][template_name] = result['data']['template'] + self._save_cache() + else: + display.vvv('lfbw - using cached template "%s"' % (template_name)) + return copy.deepcopy(self._cache['templates'][template_name]) + + + @property def is_unlocked(self): """Check if the Bitwarden vault is unlocked. """ @@ -206,10 +257,20 @@ def is_unlocked(self): return result['data']['template']['status'] == 'unlocked' - def sync(self): - """Pull the latest vault data from server. + def sync(self, force=False, interval=60): + """Pull the latest vault data from server and repopulate the items cache. + Syncs only if the last sync was more than `interval` seconds ago, unless `force` is True. """ - return self._api_call('sync', method='POST') + if not force and time.time() - self._cache.get('sync_timestamp', 0) < interval: + display.vvv('lfbw - sync skipped, last sync was recent enough') + return + display.vvv('lfbw - syncing vault (force=%s)' % (force)) + self._api_call('sync', method='POST') + result = self._api_call('list/object/items') + self._cache['items'] = result['data']['data'] + self._cache['sync_timestamp'] = time.time() + display.vvv('lfbw - sync complete, cached %d items' % (len(self._cache['items']))) + self._save_cache() def get_items(self, name, username=None, folder_id=None, collection_id=None, organization_id=None): @@ -256,18 +317,11 @@ def get_items(self, name, username=None, folder_id=None, collection_id=None, org if isinstance(organization_id, str) and len(organization_id.strip()) == 0: organization_id = None - params = urllib.parse.urlencode( - { - 'search': name, - }, - quote_via=urllib.parse.quote, - ) - result = self._api_call('list/object/items?%s' % (params)) - - # make sure that all the given parameters exactly match the requested one, as `bw` is not that precise (only performs a search) - # we are not using the filter parameters of the `bw` utility, as they perform an OR operation, but we want AND + display.vvv('lfbw - searching cache for name="%s", username="%s"' % (name, username)) matching_items = [] - for item in result['data']['data']: + for item in self._cache['items']: + if item.get('type') != 1: + continue # skip non-login items (cards, secure notes, identities) if item['name'] == name \ and (item['login']['username'] == username) \ and (item.get('folderId') == folder_id) \ @@ -280,13 +334,20 @@ def get_items(self, name, username=None, folder_id=None, collection_id=None, org and (item.get('organizationId') == organization_id): matching_items.append(item) + display.vvv('lfbw - found %d matching item(s)' % (len(matching_items))) return matching_items def get_item_by_id(self, item_id): """Get an item by ID from Bitwarden. Returns the item or None. Throws an exception if the id leads to unambiguous results. """ - + display.vvv('lfbw - looking up item by id=%s' % (item_id)) + for item in self._cache['items']: + if item.get('id') == item_id: + display.vvv('lfbw - found item in cache') + return item + # fallback to API if not found in cache (item could have been created externally) + display.vvv('lfbw - item not in cache, falling back to API') result = self._api_call('object/item/%s' % (item_id)) return result['data'] @@ -319,9 +380,7 @@ def get_template_item_login_uri(self, uris): """ login_uris = [] if uris: - # To create uris, fetch the JSON structure for that. - result = self._api_call('object/template/item.login.uri') - template = result['data']['template'] + template = self._get_template('item.login.uri') for uri in uris: login_uri = template.copy() # make sure we are not editing the same object repeatedly login_uri['uri'] = uri @@ -342,9 +401,7 @@ def get_template_item_login(self, username=None, password=None, login_uris=None) "totp": "JBSWY3DPEHPK3PXP" } """ - # To create a login item, fetch the JSON structure for that. - result = self._api_call('object/template/item.login') - login = result['data']['template'] + login = self._get_template('item.login') login['password'] = password login['totp'] = '' login['uris'] = login_uris or [] @@ -374,10 +431,7 @@ def get_template_item(self, name, login=None, notes=None, organization_id=None, "reprompt": 0 } """ - # To create an item later on, fetch the item JSON structure, and fill in the appropriate - # values. - result = self._api_call('object/template/item') - item = result['data']['template'] + item = self._get_template('item') item['collectionIds'] = collection_ids item['folderId'] = folder_id item['login'] = login @@ -391,20 +445,32 @@ def get_template_item(self, name, login=None, notes=None, organization_id=None, def create_item(self, item): """Creates an item object in Bitwarden. """ + display.vvv('lfbw - creating item "%s"' % (item.get('name', ''))) result = self._api_call('object/item', method='POST', body=item) + self._cache['items'].append(result['data']) + self._save_cache() + time.sleep(1) return result['data'] def edit_item(self, item, item_id): """Edits an item object in Bitwarden. """ + display.vvv('lfbw - editing item %s' % (item_id)) result = self._api_call('object/item/%s' % (item_id), method='PUT', body=item) + for i, cached_item in enumerate(self._cache['items']): + if cached_item.get('id') == item_id: + self._cache['items'][i] = result['data'] + break + self._save_cache() + time.sleep(1) return result['data'] def add_attachment(self, item_id, attachment_path): """Adds the file at `attachment_path` to the item specified by `item_id` """ + display.vvv('lfbw - adding attachment "%s" to item %s' % (attachment_path, item_id)) body = { 'file': { @@ -412,6 +478,12 @@ def add_attachment(self, item_id, attachment_path): }, } result = self._api_call('attachment?itemId=%s' % (item_id), method='POST', body=body, body_format='form-multipart') + for i, cached_item in enumerate(self._cache['items']): + if cached_item.get('id') == item_id: + self._cache['items'][i] = result['data'] + break + self._save_cache() + time.sleep(1) return result @staticmethod diff --git a/plugins/modules/bitwarden_item.py b/plugins/modules/bitwarden_item.py index ac0920d2d..99a5e4f09 100644 --- a/plugins/modules/bitwarden_item.py +++ b/plugins/modules/bitwarden_item.py @@ -18,8 +18,8 @@ - If no password item is found, a new item is created. Useful for automation. - If you do not specify a name or Bitwarden ID, it searches using the name/title. - If there is an existing Bitwarden item that differs from the given parameters, the item is updated, and the updated item is returned. - - If a search returns multiple entries, this lookup plugin throws an error, since it cannot decide which one to use. - - On success, this lookup plugin returns the complete Bitwarden item object. + - If a search returns multiple entries, this module throws an error, since it cannot decide which one to use. + - On success, this module returns the complete Bitwarden item object. - If you don't specify a name/title for a password item, a name/title will be created automatically, using C(hostname - purpose) (for example "C(dbserver - MariaDB)") or just C(hostname) (for example "C(dbserver)", depending on what is provided). notes: @@ -288,7 +288,7 @@ def run_module(): module_args = dict( attachments=dict(type='list', required=False, default=[]), collection_id=dict(type='str', required=False, default=None), - folder_id=dict(type='list', required=False, default=None), + folder_id=dict(type='str', required=False, default=None), hostname=dict(type='str', required=False, default=None), id=dict(type='str', required=False, default=None), name=dict(type='str', required=False, default=None),