Merge branch 'master' of github.com:scottwallacesh/dotfiles

.ansible/roles/common_server/tasks/docker.yaml

@@ -0,0 +1,22 @@
+- name: "Docker"
+  tags:
+    - docker
+  block:
+    - name: "Docker: packages"
+      tags:
+        - install
+      package:
+        name: "{{ item }}"
+        state: latest
+      loop:
+        - "docker.io"
+        - "docker-compose"
+      become: yes
+
+    - name: "Docker: directory"
+      tags:
+        - install
+      file:
+        state: directory
+        path: "/srv/app/docker"
+      become: yes

.ansible/roles/haproxy_server/files/haproxy.cfg

@@ -87,7 +87,7 @@ backend matrix-backend
     server localhost 10.0.9.111:8008 maxconn 64
 
 backend searx-backend
-    server localhost 127.0.0.1:8888
+    server localhost 10.0.9.111:8888
     acl network_allowed src 10.0.9.0/24
 
 backend share-backend

.ansible/roles/linux_desktop/vars/main.yaml

@@ -60,6 +60,7 @@ gnome_extension_ids:
   - 1206   # clock-override
   - 28     # gtile
   - 1386   # notification counter
+  - 1034   # systemd unit toggles
 
 gnome_custom_keybindings:
   - name: 'Show CopyQ clipboards'

.ansible/roles/matrix_server/files/log.yaml

@@ -1,37 +0,0 @@
-version: 1
-
-formatters:
-  precise:
-    format: "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s"
-
-filters:
-  context:
-    (): synapse.logging.context.LoggingContextFilter
-    request: ""
-
-handlers:
-  file:
-    class: logging.handlers.RotatingFileHandler
-    formatter: precise
-    filename: /var/log/matrix-synapse/homeserver.log
-    maxBytes: 104857600
-    backupCount: 10
-    filters: [context]
-    encoding: utf8
-  console:
-    class: logging.StreamHandler
-    formatter: precise
-    level: WARN
-
-loggers:
-  synapse:
-    level: WARN
-
-  synapse.storage.SQL:
-    # beware: increasing this to DEBUG will make synapse log sensitive
-    # information such as access tokens.
-    level: WARN
-
-root:
-  level: WARN
-  handlers: [file, console]

.ansible/roles/matrix_server/tasks/includes/repos.yaml

@@ -1,19 +0,0 @@
-- name: includes/repos.yaml
-  tags:
-    - install
-    - repos
-    - matrix
-  block:
-    - name: "Apt repo: Import GPG key: {{ repo.name }}"
-      apt_key:
-        state: present
-        url: "{{ repo.key }}"
-      when: repo.key
-      become: true
-
-    - name: "Apt repo: Add repo: {{ repo.name }}"
-      apt_repository:
-        state: present
-        repo: "{{ repo.data }}"
-        update_cache: yes
-      become: true

.ansible/roles/matrix_server/tasks/main.yaml

@@ -1,60 +1,14 @@
-- name: "Matrix: Repositories"
-  tags:
-    - install
-    - repos
-    - matrix
-  include: includes/repos.yaml
-  loop: "{{ linux_repos }}"
-  loop_control:
-    loop_var: repo
-
-- name: "Matrix: Packages"
-  tags:
-    - install
-    - packages
-    - matrix
-  package:
-    name: "{{ item }}"
-    state: present
-  loop: "{{ linux_packages }}"
-  become: true
-
 - name: "Matrix: Configuration"
   tags:
     - config
     - matrix
   block:
+    - include: roles/common_server/tasks/docker.yaml
+
     - name: "Matrix: Configuration: homeserver.yaml"
       template:
         src: "templates/homeserver.yaml.j2"
-        dest: "/etc/matrix-synapse/homeserver.yaml"
+        dest: "/srv/app/docker/matrix/config/homeserver.yaml"
-        mode: "0400"
-        owner: "matrix-synapse"
-        group: "nogroup"
-      become: yes
-
-    - name: "Matrix: Configuration: server_name.yaml"
-      copy:
-        dest: "/etc/matrix-synapse/conf.d/server_name.yaml"
-        content: "server_name: home.suborbit.com"
-        mode: "0400"
-        owner: "matrix-synapse"
-        group: "nogroup"
-      become: yes
-
-    - name: "Matrix: Configuration: log.yaml"
-      copy:
-        dest: "/etc/matrix-synapse/log.yaml"
-        src: "files/log.yaml"
-        mode: "0400"
-        owner: "matrix-synapse"
-        group: "nogroup"
-      become: yes
-
-    - name: "Matrix: Configuration: dhparam.pem"
-      copy:
-        dest: "/etc/matrix-synapse/dhparam.pem"
-        content: "{{ dhparam }}"
         mode: "0400"
         owner: "matrix-synapse"
         group: "nogroup"
@@ -62,18 +16,16 @@
 
     - name: "Matrix: Configuration: homeserver.signing.key"
       copy:
-        dest: "/etc/matrix-synapse/homeserver.signing.key"
+        dest: "/srv/app/docker/matrix/config/homeserver.signing.key"
         content: "{{ signing_key }}"
         mode: "0400"
         owner: "matrix-synapse"
         group: "nogroup"
       become: yes
 
-- name: "Matrix: Service"
+    - name: "Matrix: Docker: compose.yaml"
-  tags:
+      copy:
-    - matrix
+        dest: "/srv/app/docker/matrix/compose.yaml"
-  systemd:
+        src: "files/docker-compose.yaml"
-    name: "matrix-synapse"
+        mode: "0400"
-    state: started
+      become: yes
-    enabled: yes
-  become: yes

.ansible/roles/matrix_server/templates/homeserver.yaml.j2

@@ -3,6 +3,7 @@ tls_fingerprints: [{ "sha256": "/HCvvvL0fZZb3BsgA8KIegBijVjk4UCbA9od18BLxOE" }]
 pid_file: /run/matrix-synapse.pid
 soft_file_limit: 0
 use_presence: true
+server_name: home.suborbit.com
 listeners:
   - port: 8008
     tls: false
@@ -17,9 +18,8 @@ listeners:
 database:
   name: "sqlite3"
   args:
-    database: "/var/lib/matrix-synapse/homeserver.db"
+    database: "/data/homeserver.db"
 event_cache_size: "10K"
-log_config: "/etc/matrix-synapse/log.yaml"
 rc_messages_per_second: 0.2
 rc_message_burst_count: 10.0
 federation_rc_window_size: 1000
@@ -27,8 +27,8 @@ federation_rc_sleep_limit: 10
 federation_rc_sleep_delay: 500
 federation_rc_reject_limit: 5
 federation_rc_concurrent: 1
-media_store_path: "/var/lib/matrix-synapse/media"
+media_store_path: "/data/media"
-uploads_path: "/var/lib/matrix-synapse/uploads"
+uploads_path: "/data/uploads"
 max_upload_size: "100M"
 max_image_pixels: "64M"
 dynamic_thumbnails: false
@@ -76,7 +76,7 @@ room_invite_state_types:
 app_service_config_files: []
 track_appservice_user_ips: False
 expire_access_token: False
-signing_key_path: "/etc/matrix-synapse/homeserver.signing.key"
+signing_key_path: "/config/homeserver.signing.key"
 old_signing_keys: {}
 key_refresh_interval: "1d" # 1 Day.
 trusted_key_servers:

.ansible/roles/matrix_server/vars/main.yaml

@@ -1,12 +1,4 @@
 ---
-linux_repos:
-  - name: "Matrix Synapse"
-    data: "deb https://packages.matrix.org/debian disco main"
-    key: "https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg"
-
-linux_packages:
-  - matrix-synapse-py3
-
 password_pepper: !vault |
   $ANSIBLE_VAULT;1.1;AES256
   33373335643631373939643231653138646134633537343138333461633164393764376236663065

.ansible/roles/media_server/files/cloudplow.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=cloudplow
-After=mnt-media.mount
-Requires=mnt-media.mount
-
-[Service]
-User=media
-Group=media
-Type=simple
-WorkingDirectory=/srv/app/cloudplow/
-ExecStart=/srv/app/cloudplow/.venv/bin/python3 /srv/app/cloudplow/cloudplow.py run --loglevel=INFO
-ExecStopPost=/bin/rm -rf /srv/app/cloudplow/locks
-Restart=always
-RestartSec=10
-
-[Install]
-WantedBy=default.target

.ansible/roles/media_server/files/cloudplow_config.json

@@ -1,61 +0,0 @@
-{
-  "core": {
-    "dry_run": false,
-    "rclone_binary_path": "/usr/bin/rclone",
-    "rclone_config_path": "/srv/etc/rclone/rclone.conf"
-  },
-  "hidden": {
-    "/mnt/.media.cache/.unionfs": {
-      "hidden_remotes": [
-        "google"
-      ]
-    }
-  },
-  "notifications": {},
-  "remotes": {
-    "google": {
-      "hidden_remote": "GoogleDriveCrypt:",
-      "rclone_command": "move",
-      "rclone_excludes": [
-        "**partial~",
-        "**_HIDDEN~",
-        ".unionfs/**"
-      ],
-      "rclone_extras": {
-        "--bwlimit": "23:00,off 07:00,1M",
-        "--checkers": 16,
-        "--drive-chunk-size": "64M",
-        "--skip-links": null,
-        "--stats": "60s",
-        "--transfers": 4,
-        "--verbose": 1
-      },
-      "rclone_sleeps": {
-        "Failed to copy: googleapi: Error 403: User rate limit exceeded": {
-          "count": 5,
-          "sleep": 25,
-          "timeout": 3600
-        }
-      },
-      "remove_empty_dir_depth": 2,
-      "sync_remote": "GoogleDriveCrypt:",
-      "upload_folder": "/mnt/.media.cache",
-      "upload_remote": "GoogleDriveCloudplowUpload:"
-    }
-  },
-  "syncer": {},
-  "uploader": {
-    "google": {
-      "check_interval": 1,
-      "exclude_open_files": true,
-      "max_size_gb": 10,
-      "opened_excludes": [],
-      "schedule": {
-        "allowed_from": "04:00",
-        "allowed_until": "08:00",
-        "enabled": false
-      },
-      "size_excludes": []
-    }
-  }
-}

.ansible/roles/media_server/files/docker-compose.yaml

@@ -62,7 +62,7 @@ services:
     volumes:
       - /srv/app/docker/mediaserver/config/radarr:/config
       - /mnt/media/_Movies:/movies
-      - /srv/tmp/downloads/complete/movies:/downloads
+      - /srv/tmp/downloads:/downloads
     ports:
       - 7878:7878
     restart: unless-stopped
@@ -77,7 +77,7 @@ services:
       - UMASK_SET=022
     volumes:
       - /srv/app/docker/mediaserver/config/sabnzbd:/config
-      - /srv/tmp/downloads/complete:/downloads
+      - /srv/tmp/downloads:/downloads
       - /srv/tmp/downloads/incomplete:/incomplete-downloads
     ports:
       - 8080:8080
@@ -95,7 +95,22 @@ services:
     volumes:
       - /srv/app/docker/mediaserver/config/sonarr:/config
       - /mnt/media/_TV:/tv
-      - /srv/tmp/downloads/complete/tv:/downloads
+      - /srv/tmp/downloads:/downloads
     ports:
       - 8989:8989
     restart: unless-stopped
+
+  cloudplow:
+    image: sabrsorensen/alpine-cloudplow
+    container_name: cloudplow
+    environment:
+      - PUID=1003
+      - PGID=1003
+      - TZ=Europe/London
+      - UMASK_SET=022
+    volumes:
+      - /srv/app/docker/mediaserver/config/cloudplow:/config:rw
+      - /srv/etc/rclone:/config/rclone:rw
+      - /mnt/.media.cache:/data/imported_media:rw
+      - /etc/localtime:/etc/localtime:ro
+    restart: unless-stopped

.ansible/roles/media_server/tasks/main.yaml

@@ -2,10 +2,11 @@
   tags:
     - mediaserver
   block:
+    - include: roles/common_server/tasks/docker.yaml
+
     - name: "Media Server: Group"
       tags:
         - install
-        - cloudplow
       group:
         name: "media"
       become: yes
@@ -13,7 +14,6 @@
     - name: "Media Server: User"
       tags:
         - install
-        - cloudplow
       user:
         name: "media"
         group: "media"
@@ -21,100 +21,24 @@
         expires: -1
       become: yes
 
-    - name: "Media Server: Cloudplow: git repo"
+    - name: "Media Server: directories"
       tags:
-        - install
+        - config
-        - cloudplow
-      git:
-        repo: "https://github.com/l3uddz/cloudplow.git"
-        dest: "/srv/app/cloudplow"
-      become: yes
-
-    - name: "Media Server: Cloudplow: directory permissions"
-      tags:
-        - install
-        - cloudplow
       file:
-        path: "/srv/app/cloudplow"
+        path: "{{ item }}"
         state: directory
-        mode: "u+w"
         owner: "media"
         group: "media"
-        recurse: yes
-      become: yes
-
-    - name: "Media Server: Cloudplow: config"
-      tags:
-        - config
-        - cloudplow
-      copy:
-        src: "files/cloudplow_config.json"
-        dest: "/srv/app/cloudplow/config.json"
-        mode: "0600"
-        owner: "media"
-        group: "media"
-      become: yes
-
-    - name: "Media Server: Cloudplow: Python dependencies"
-      tags:
-        - install
-        - cloudplow
-      pip:
-        requirements: "/srv/app/cloudplow/requirements.txt"
-        virtualenv: "/srv/app/cloudplow/.venv"
-        virtualenv_python: python3
-      become: yes
-
-    - name: "Media Server: Cloudplow: service config"
-      tags:
-        - install
-        - cloudplow
-      copy:
-        dest: "/etc/systemd/system/cloudplow.service"
-        src: "files/cloudplow.service"
-        mode: "0644"
-      become: yes
-
-    - name: "Media Server: Cloudplow: service"
-      tags:
-        - config
-        - cloudplow
-      systemd:
-        name: "cloudplow"
-        state: started
-        enabled: yes
-        daemon_reload: yes
-      become: yes
-
-    - name: "Media Server: Docker: packages"
-      tags:
-        - install
-        - docker
-      package:
-        name: "{{ item }}"
-        state: latest
       loop:
-        - "docker.io"
+        - "/srv/app/docker/mediaserver"
-        - "docker-compose"
+        - "/srv/app/docker/mediaserver/config"
       become: yes
 
-    - name: "Media Server: Docker: directories"
+    - name: "Media Server: Docker: compose.yaml"
       tags:
         - config
-        - docker
-      file:
-        path: "/srv/app/docker/mediaserver/config"
-        state: directory
-        owner: "media"
-        group: "media"
-        recurse: yes
-      become: yes
-
-    - name: "Media Server: Docker: config"
-      tags:
-        - config
-        - docker
       copy:
         dest: "/srv/app/docker/mediaserver/compose.yaml"
         src: "files/docker-compose.yaml"
+        mode: "0400"
       become: yes

.ansible/roles/rclone_mount/files/rclone.service

@@ -13,10 +13,8 @@ ExecStart=/usr/bin/rclone mount \
         --allow-other \
         --no-modtime \
         --read-only \
-        --dir-cache-time=240m \
         --tpslimit=10 \
         --tpslimit-burst=1 \
-        --buffer-size=1G \
         GoogleDriveCrypt:/ /mnt/GoogleDriveCrypt
 ExecStartPost=/bin/bash -c 'while [ ! -d /mnt/GoogleDriveCrypt/_Other ]; do /usr/bin/sleep 5; done'
 ExecStop=/bin/fusermount -u /mnt/GoogleDriveCrypt

.ansible/roles/searx_server/files/docker-compose.yaml

@@ -0,0 +1,21 @@
+---
+version: "2"
+services:
+  searx:
+    image: hoellen/searx
+    container_name: searx
+    environment:
+      - UID=1002
+      - GID=1002
+      - TZ=Europe/London
+      - UMASK_SET=022
+    volumes:
+      - /srv/app/docker/searx/config/settings.yml:/usr/local/searx/searx/settings.yml:ro
+      - /srv/app/docker/searx/config/plugins/open_results_on_new_tab.py:/usr/local/searx/searx/plugins/open_results_on_new_tab.py:ro
+      - /srv/app/docker/searx/config/plugins/oa_doi_rewrite.py:/usr/local/searx/searx/plugins/oa_doi_rewrite.py:ro
+      - /srv/app/docker/searx/config/plugins/infinite_scroll.py:/usr/local/searx/searx/plugins/infinite_scroll.py:ro
+      - /srv/app/docker/searx/config/engines/google.py:/usr/local/searx/searx/engines/google.py:ro
+
+    ports:
+      - 8888:8888
+    restart: unless-stopped

.ansible/roles/searx_server/files/engines/google.py

@@ -0,0 +1,443 @@
+#  Google (Web)
+#
+# @website     https://www.google.com
+# @provide-api yes (https://developers.google.com/custom-search/)
+#
+# @using-api   no
+# @results     HTML
+# @stable      no (HTML can change)
+# @parse       url, title, content, suggestion
+
+import re
+from flask_babel import gettext
+from lxml import html, etree
+from searx.engines.xpath import extract_text, extract_url
+from searx import logger
+from searx.url_utils import urlencode, urlparse, parse_qsl
+from searx.utils import match_language
+
+logger = logger.getChild('google engine')
+
+
+# engine dependent config
+categories = ['general']
+paging = True
+language_support = True
+use_locale_domain = True
+time_range_support = True
+
+# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
+default_hostname = 'www.google.com'
+
+country_to_hostname = {
+    'BG': 'www.google.bg',  # Bulgaria
+    'CZ': 'www.google.cz',  # Czech Republic
+    'DE': 'www.google.de',  # Germany
+    'DK': 'www.google.dk',  # Denmark
+    'AT': 'www.google.at',  # Austria
+    'CH': 'www.google.ch',  # Switzerland
+    'GR': 'www.google.gr',  # Greece
+    'AU': 'www.google.com.au',  # Australia
+    'CA': 'www.google.ca',  # Canada
+    'GB': 'www.google.co.uk',  # United Kingdom
+    'ID': 'www.google.co.id',  # Indonesia
+    'IE': 'www.google.ie',  # Ireland
+    'IN': 'www.google.co.in',  # India
+    'MY': 'www.google.com.my',  # Malaysia
+    'NZ': 'www.google.co.nz',  # New Zealand
+    'PH': 'www.google.com.ph',  # Philippines
+    'SG': 'www.google.com.sg',  # Singapore
+    # 'US': 'www.google.us',  # United States, redirect to .com
+    'ZA': 'www.google.co.za',  # South Africa
+    'AR': 'www.google.com.ar',  # Argentina
+    'CL': 'www.google.cl',  # Chile
+    'ES': 'www.google.es',  # Spain
+    'MX': 'www.google.com.mx',  # Mexico
+    'EE': 'www.google.ee',  # Estonia
+    'FI': 'www.google.fi',  # Finland
+    'BE': 'www.google.be',  # Belgium
+    'FR': 'www.google.fr',  # France
+    'IL': 'www.google.co.il',  # Israel
+    'HR': 'www.google.hr',  # Croatia
+    'HU': 'www.google.hu',  # Hungary
+    'IT': 'www.google.it',  # Italy
+    'JP': 'www.google.co.jp',  # Japan
+    'KR': 'www.google.co.kr',  # South Korea
+    'LT': 'www.google.lt',  # Lithuania
+    'LV': 'www.google.lv',  # Latvia
+    'NO': 'www.google.no',  # Norway
+    'NL': 'www.google.nl',  # Netherlands
+    'PL': 'www.google.pl',  # Poland
+    'BR': 'www.google.com.br',  # Brazil
+    'PT': 'www.google.pt',  # Portugal
+    'RO': 'www.google.ro',  # Romania
+    'RU': 'www.google.ru',  # Russia
+    'SK': 'www.google.sk',  # Slovakia
+    'SI': 'www.google.si',  # Slovenia
+    'SE': 'www.google.se',  # Sweden
+    'TH': 'www.google.co.th',  # Thailand
+    'TR': 'www.google.com.tr',  # Turkey
+    'UA': 'www.google.com.ua',  # Ukraine
+    # 'CN': 'www.google.cn',  # China, only from China ?
+    'HK': 'www.google.com.hk',  # Hong Kong
+    'TW': 'www.google.com.tw'  # Taiwan
+}
+
+# osm
+url_map = 'https://www.openstreetmap.org/'\
+    + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
+
+# search-url
+search_path = '/search'
+search_url = ('https://{hostname}' +
+              search_path +
+              '?{query}&start={offset}&gws_rd=cr&gbv=1&lr={lang}&hl={lang_short}&ei=x')
+
+time_range_search = "&tbs=qdr:{range}"
+time_range_dict = {'day': 'd',
+                   'week': 'w',
+                   'month': 'm',
+                   'year': 'y'}
+
+# other URLs
+map_hostname_start = 'maps.google.'
+maps_path = '/maps'
+redirect_path = '/url'
+images_path = '/images'
+supported_languages_url = 'https://www.google.com/preferences?#languages'
+
+# specific xpath variables
+results_xpath = '//div[@class="g"]'
+url_xpath = './/h3/a/@href'
+title_xpath = './/h3'
+content_xpath = './/span[@class="st"]'
+content_misc_xpath = './/div[@class="f slp"]'
+suggestion_xpath = '//p[@class="_Bmc"]'
+spelling_suggestion_xpath = '//a[@class="spell"]'
+
+# map : detail location
+map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
+map_phone_xpath = './/div[@class="s"]//table//td[2]/span/span'
+map_website_url_xpath = 'h3[2]/a/@href'
+map_website_title_xpath = 'h3[2]'
+
+# map : near the location
+map_near = 'table[@class="ts"]//tr'
+map_near_title = './/h4'
+map_near_url = './/h4/a/@href'
+map_near_phone = './/span[@class="nobr"]'
+
+# images
+images_xpath = './/div/a'
+image_url_xpath = './@href'
+image_img_src_xpath = './img/@src'
+
+# property names
+# FIXME : no translation
+property_address = "Address"
+property_phone = "Phone number"
+
+
+# remove google-specific tracking-url
+def parse_url(url_string, google_hostname):
+    # sanity check
+    if url_string is None:
+        return url_string
+
+    # normal case
+    parsed_url = urlparse(url_string)
+    if (parsed_url.netloc in [google_hostname, '']
+            and parsed_url.path == redirect_path):
+        query = dict(parse_qsl(parsed_url.query))
+        return query['q']
+    else:
+        return url_string
+
+
+# returns extract_text on the first result selected by the xpath or None
+def extract_text_from_dom(result, xpath):
+    r = result.xpath(xpath)
+    if len(r) > 0:
+        return extract_text(r[0])
+    return None
+
+
+# do search-request
+def request(query, params):
+    offset = (params['pageno'] - 1) * 10
+
+    if params['language'] == 'all' or params['language'] == 'en-US':
+        language = 'en-GB'
+    else:
+        language = match_language(params['language'], supported_languages, {})
+
+    language_array = language.split('-')
+    if params['language'].find('-') > 0:
+        country = params['language'].split('-')[1]
+    elif len(language_array) == 2:
+        country = language_array[1]
+    else:
+        country = 'US'
+
+    url_lang = 'lang_' + language
+
+    if use_locale_domain:
+        google_hostname = country_to_hostname.get(country.upper(), default_hostname)
+    else:
+        google_hostname = default_hostname
+
+    # original format: ID=3e2b6616cee08557:TM=5556667580:C=r:IP=4.1.12.5-:S=23ASdf0soFgF2d34dfgf-_22JJOmHdfgg
+    params['cookies']['GOOGLE_ABUSE_EXEMPTION'] = 'x'
+    params['url'] = search_url.format(offset=offset,
+                                      query=urlencode({'q': query}),
+                                      hostname=google_hostname,
+                                      lang=url_lang,
+                                      lang_short=language)
+    if params['time_range'] in time_range_dict:
+        params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])
+
+    params['headers']['Accept-Language'] = language + ',' + language + '-' + country
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+
+    # Force Internet Explorer 12 user agent to avoid loading the new UI that Searx can't parse
+    params['headers']['User-Agent'] = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"
+
+    params['google_hostname'] = google_hostname
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    # detect google sorry
+    resp_url = urlparse(resp.url)
+    if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
+        raise RuntimeWarning('sorry.google.com')
+
+    if resp_url.path.startswith('/sorry'):
+        raise RuntimeWarning(gettext('CAPTCHA required'))
+
+    # which hostname ?
+    google_hostname = resp.search_params.get('google_hostname')
+    google_url = "https://" + google_hostname
+
+    # convert the text to dom
+    dom = html.fromstring(resp.text)
+
+    instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
+    if instant_answer:
+        results.append({'answer': u' '.join(instant_answer)})
+    try:
+        results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
+                          .split()[1].replace(',', ''))
+        results.append({'number_of_results': results_num})
+    except:
+        pass
+
+    # parse results
+    for result in dom.xpath(results_xpath):
+        try:
+            title = extract_text(result.xpath(title_xpath)[0])
+            url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)
+            parsed_url = urlparse(url, google_hostname)
+
+            # map result
+            if parsed_url.netloc == google_hostname:
+                # TODO fix inside links
+                continue
+                # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
+                #     print "yooooo"*30
+                #     x = result.xpath(map_near)
+                #     if len(x) > 0:
+                #         # map : near the location
+                #         results = results + parse_map_near(parsed_url, x, google_hostname)
+                #     else:
+                #         # map : detail about a location
+                #         results = results + parse_map_detail(parsed_url, result, google_hostname)
+                # # google news
+                # elif parsed_url.path == search_path:
+                #     # skipping news results
+                #     pass
+
+                # # images result
+                # elif parsed_url.path == images_path:
+                #     # only thumbnail image provided,
+                #     # so skipping image results
+                #     # results = results + parse_images(result, google_hostname)
+                #     pass
+
+            else:
+                # normal result
+                content = extract_text_from_dom(result, content_xpath)
+                if content is None:
+                    continue
+                content_misc = extract_text_from_dom(result, content_misc_xpath)
+                if content_misc is not None:
+                    content = content_misc + "<br />" + content
+                # append result
+                results.append({'url': url,
+                                'title': title,
+                                'content': content
+                                })
+        except:
+            logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))
+            continue
+
+    logger.debug(results)
+    if not results:
+        logger.debug('SCOTT trying new interface')
+        a_tags = dom.xpath('//a')
+        for a_tag in a_tags:
+            href = a_tag.get('href', '')
+            url = dict(parse_qsl(href)).get('/url?q', None)
+            if not (url and url.startswith('http')):
+                continue
+            ancestors = [x for x in a_tag.iterancestors()]
+            try:
+                title_divs = a_tag.xpath('./div')
+                if title_divs:
+                    title = title_divs[0].text.strip().replace('\n', ' ').replace('  ', ' ')
+                else:
+                    title = None
+                item_div = ancestors[2]
+                content_divs = item_div.xpath('./div/div[3]/div/div/div/div/div[1]/div')
+                if content_divs:
+                    content = content_divs[0].text.strip()
+                else:
+                    content_divs = item_div.xpath('./div/div[3]/div/div/div/div/div')
+                    if content_divs:
+                        content = content_divs[0].text.strip().replace('\n', ' ').replace('  ', ' ')
+                    else:
+                        content = None
+                if not content:
+                    content_divs = ancestors[1].xpath('./div/div/div/div/div/div')
+                    if content_divs:
+                        content = extract_text(content_divs[0])
+                    pass
+                if title or content:
+                    if any(x['url'] == url for x in results):
+                        results_item = results.pop(
+                            results.index(
+                                [x for x in results if x['url'] == url][0]))
+                        for key, var in (('title', title), ('content', content)):
+                            if not results_item[key]:
+                                results_item[key] = var
+                        results.append(results_item)
+                    else:
+                        results.append({
+                            'url': url, 'title': title, 'content': content})
+            except Exception as err:
+                logger.error(err, exc_info=1)
+
+    # parse suggestion
+    for suggestion in dom.xpath(suggestion_xpath):
+        # append suggestion
+        results.append({'suggestion': extract_text(suggestion)})
+
+    for correction in dom.xpath(spelling_suggestion_xpath):
+        results.append({'correction': extract_text(correction)})
+
+    # return results
+    return results
+
+
+def parse_images(result, google_hostname):
+    results = []
+    for image in result.xpath(images_xpath):
+        url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)
+        img_src = extract_text(image.xpath(image_img_src_xpath)[0])
+
+        # append result
+        results.append({'url': url,
+                        'title': '',
+                        'content': '',
+                        'img_src': img_src,
+                        'template': 'images.html'
+                        })
+
+    return results
+
+
+def parse_map_near(parsed_url, x, google_hostname):
+    results = []
+
+    for result in x:
+        title = extract_text_from_dom(result, map_near_title)
+        url = parse_url(extract_text_from_dom(result, map_near_url), google_hostname)
+        attributes = []
+        phone = extract_text_from_dom(result, map_near_phone)
+        add_attributes(attributes, property_phone, phone, 'tel:' + phone)
+        results.append({'title': title,
+                        'url': url,
+                        'content': attributes_to_html(attributes)
+                        })
+
+    return results
+
+
+def parse_map_detail(parsed_url, result, google_hostname):
+    results = []
+
+    # try to parse the geoloc
+    m = re.search(r'@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path)
+    if m is None:
+        m = re.search(r'll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query)
+
+    if m is not None:
+        # geoloc found (ignored)
+        lon = float(m.group(2))  # noqa
+        lat = float(m.group(1))  # noqa
+        zoom = int(m.group(3))  # noqa
+
+        # attributes
+        attributes = []
+        address = extract_text_from_dom(result, map_address_xpath)
+        phone = extract_text_from_dom(result, map_phone_xpath)
+        add_attributes(attributes, property_address, address, 'geo:' + str(lat) + ',' + str(lon))
+        add_attributes(attributes, property_phone, phone, 'tel:' + phone)
+
+        # title / content / url
+        website_title = extract_text_from_dom(result, map_website_title_xpath)
+        content = extract_text_from_dom(result, content_xpath)
+        website_url = parse_url(extract_text_from_dom(result, map_website_url_xpath), google_hostname)
+
+        # add a result if there is a website
+        if website_url is not None:
+            results.append({'title': website_title,
+                            'content': (content + '<br />' if content is not None else '')
+                            + attributes_to_html(attributes),
+                            'url': website_url
+                            })
+
+    return results
+
+
+def add_attributes(attributes, name, value, url):
+    if value is not None and len(value) > 0:
+        attributes.append({'label': name, 'value': value, 'url': url})
+
+
+def attributes_to_html(attributes):
+    retval = '<table class="table table-striped">'
+    for a in attributes:
+        value = a.get('value')
+        if 'url' in a:
+            value = '<a href="' + a.get('url') + '">' + value + '</a>'
+        retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>'
+    retval = retval + '</table>'
+    return retval
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = {}
+    dom = html.fromstring(resp.text)
+    options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
+    for option in options:
+        code = option.xpath('./@value')[0].split('_')[-1]
+        name = option.xpath('./@data-name')[0].title()
+        supported_languages[code] = {"name": name}
+
+    return supported_languages

.ansible/roles/searx_server/files/plugins/infinite_scroll.py

@@ -0,0 +1,9 @@
+from flask_babel import gettext
+
+name = gettext('Infinite scroll')
+description = gettext('Automatically load next page when scrolling to bottom of current page')
+default_on = True
+preference_section = 'ui'
+
+js_dependencies = ('plugins/js/infinite_scroll.js',)
+css_dependencies = ('plugins/css/infinite_scroll.css',)

.ansible/roles/searx_server/files/plugins/oa_doi_rewrite.py

@@ -0,0 +1,45 @@
+from flask_babel import gettext
+import re
+from searx.url_utils import urlparse, parse_qsl
+from searx import settings
+
+
+regex = re.compile(r'10\.\d{4,9}/[^\s]+')
+
+name = gettext('Open Access DOI rewrite')
+description = gettext('Avoid paywalls by redirecting to open-access versions of publications when available')
+default_on = True
+preference_section = 'general'
+
+doi_resolvers = settings['doi_resolvers']
+
+
+def extract_doi(url):
+    match = regex.search(url.path)
+    if match:
+        return match.group(0)
+    for _, v in parse_qsl(url.query):
+        match = regex.search(v)
+        if match:
+            return match.group(0)
+    return None
+
+
+def get_doi_resolver(args, preference_doi_resolver):
+    doi_resolvers = settings['doi_resolvers']
+    doi_resolver = args.get('doi_resolver', preference_doi_resolver)[0]
+    if doi_resolver not in doi_resolvers:
+        doi_resolvers = settings['default_doi_resolver']
+    doi_resolver_url = doi_resolvers[doi_resolver]
+    return doi_resolver_url
+
+
+def on_result(request, search, result):
+    doi = extract_doi(result['parsed_url'])
+    if doi and len(doi) < 50:
+        for suffix in ('/', '.pdf', '/full', '/meta', '/abstract'):
+            if doi.endswith(suffix):
+                doi = doi[:-len(suffix)]
+        result['url'] = get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')) + doi
+        result['parsed_url'] = urlparse(result['url'])
+    return True

.ansible/roles/searx_server/files/plugins/open_results_on_new_tab.py

@@ -0,0 +1,25 @@
+'''
+searx is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+searx is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with searx. If not, see < http://www.gnu.org/licenses/ >.
+
+(C) 2016 by Adam Tauber, <asciimoo@gmail.com>
+'''
+from flask_babel import gettext
+name = gettext('Open result links on new browser tabs')
+description = gettext('Results are opened in the same window by default. '
+                      'This plugin overwrites the default behaviour to open links on new tabs/windows. '
+                      '(JavaScript required)')
+default_on = True
+preference_section = 'ui'
+
+js_dependencies = ('plugins/js/open_results_on_new_tab.js',)

.ansible/roles/searx_server/files/uwsgi.ini

@@ -1,28 +0,0 @@
-[uwsgi]
-# Who will run the code
-uid = www-data
-gid = www-data
-
-# disable logging for privacy
-#disable-logging = true
-
-# Number of workers (usually CPU count)
-workers = 4
-
-# The right granted on the created socket
-chmod-socket = 666
-http-socket = [::]:8888
-
-# Plugin to use and interpretor config
-single-interpreter = true
-master = true
-plugin = python
-enable-threads = true
-lazy-apps = true
-
-# Module to import
-module = searx.webapp
-
-virtualenv = /srv/app/searx/searx-ve/
-pythonpath = /srv/app/searx/
-chdir = /srv/app/searx/searx/

.ansible/roles/searx_server/tasks/main.yaml

@@ -1,45 +1,47 @@
-- name: 'Searx'
+- name: "Searx"
   tags:
-    - install
     - searx
   block:
-    - name: 'Searx: package'
+    - include: roles/common_server/tasks/docker.yaml
-      package:
-        name: "{{ item }}"
-        state: latest
-      become: yes
-      loop:
-        - uwsgi
-        - uwsgi-plugin-python3
 
-    - name: 'Searx: Config directory'
+    - name: "Searx: Config directory"
+      tags:
+        - install
       file:
         state: directory
-        path: '/srv/app/searx/searx'
+        path: "{{ item }}"
+      loop:
+        - "/srv/app/docker/searx"
+        - "/srv/app/docker/searx/config"
+        - "/srv/app/docker/searx/config/plugins"
+        - "/srv/app/docker/searx/config/engines"
       become: yes
 
-    - name: 'Searx: Config'
+    - name: "Searx: Config: main"
+      tags:
+        - config
+      template:
+        dest: "/srv/app/docker/searx/config/settings.yml"
+        src: "templates/settings.yaml.j2"
+        mode: "0400"
+      become: yes
+
+    - name: "Searx: Config: overrides"
+      tags:
+        - config
       copy:
-        dest: '/srv/app/searx/searx/settings.yml'
+        dest: "/srv/app/docker/searx/config/{{ item }}/"
-        src: 'files/settings.yaml'
+        src: "files/{{ item }}/"
+      loop:
+        - "plugins"
+        - "engines"
       become: yes
 
-    - name: 'Searx: UWSGI integration'
+    - name: "Searx: Docker: compose.yaml"
+      tags:
+        - config
       copy:
-        dest: '/etc/uwsgi/apps-available/searx.ini'
+        dest: "/srv/app/docker/searx/compose.yaml"
-        src: 'files/uwsgi.ini'
+        src: "files/docker-compose.yaml"
-      become: yes
+        mode: "0400"
-
-    - name: 'Searx: UWSGI enable'
-      file:
-        dest: '/etc/uwsgi/apps-enabled/searx.ini'
-        src: '/etc/uwsgi/apps-available/searx.ini'
-        state: link
-      become: yes
-
-    - name: 'Searx: UWSGI service'
-      systemd:
-        name: 'uwsgi'
-        state: started
-        enabled: yes
       become: yes

.ansible/roles/searx_server/templates/settings.yaml.j2

@@ -5,21 +5,21 @@ general:
 search:
     safe_search : 1 # Filter results. 0: None, 1: Moderate, 2: Strict
     autocomplete : "google" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
-    language : "en-GB"
+    language : "en-US"
     ban_time_on_fail : 5 # ban time in seconds after engine errors
     max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
 
 server:
-    port : 8889
+    port : 8888
-    bind_address : "::" # address to listen on
+    bind_address : "0.0.0.0" # address to listen on
-    secret_key : "notused"
+    secret_key : "{{ secret }}"
     base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/"
     image_proxy : False # Proxying image results through searx
     http_protocol_version : "1.0"  # 1.0 and 1.1 are supported
 
 ui:
-    static_path : "/srv/app/searx/searx/static" # Custom static path - leave it blank if you didn't change
+    static_path : "/usr/local/searx/searx/static" # Custom static path - leave it blank if you didn't change
-    templates_path : "/srv/app/searx/searx/templates" # Custom templates path - leave it blank if you didn't change
+    templates_path : "/usr/local/searx/searx/templates" # Custom templates path - leave it blank if you didn't change
     default_theme : oscar # ui theme
     default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
     theme_args :
@@ -52,12 +52,6 @@ outgoing: # communication with search engines
 #        - 1.1.1.2
 
 engines:
-  - name: apk mirror
-    engine: apkmirror
-    timeout: 4.0
-    shortcut: apkm
-    disabled: True
-
   - name : arch linux wiki
     engine : archlinux
     shortcut : al
@@ -394,12 +388,6 @@ engines:
     timeout : 6.0
     disabled : True
 
-  - name : invidious
-    engine : invidious
-    base_url : 'https://invidio.us/'
-    shortcut: iv
-    timeout : 5.0
-
   - name: kickass
     engine : kickass
     shortcut : kc
@@ -605,7 +593,7 @@ engines:
     engine : startpage
     shortcut : sp
     timeout : 6.0
-    disabled : False
+    disabled : True
 
   - name : tokyotoshokan
     engine : tokyotoshokan
@@ -639,11 +627,6 @@ engines:
 #    content_xpath : //*[@class="meaning"]
 #    shortcut : ud
 
-  - name : unsplash
-    engine : unsplash
-    disabled: True
-    shortcut : us
-
   - name : yahoo
     engine : yahoo
     shortcut : yh

.ansible/roles/searx_server/vars/main.yaml

@@ -0,0 +1,8 @@
+secret: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  62623164336563316535333233306263613666643734616665396236353432343562313230663161
+  3833623732373932633366303533366633346133313935300a306131383764306637373738333039
+  36376462303161623430613734303234353731303664313138356432333732396661303832636263
+  3263646364323234620a316539313161303539376337653565656239636666353434306533656535
+  33346334386134643830653762663130316363653136303933306661313933613964326663366531
+  3036366164393838353034336339343234643764636438353834