initial

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, E266, E501, W503
+max-line-length = 80
+max-complexity = 18
+select = B,C,E,F,W,T4,B9

.gitignore

@@ -0,0 +1,6 @@
+venv/*
+*.egg-info
+.mypy_cache
+*.pyc
+.eggs
+docs/_build/*

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+-   repo: local
+    hooks:
+    - id: black
+      name: black
+      language: system
+      entry: python3 -m black
+      types: [python]
+    - id: flake8
+      name: flake8
+      language: system
+      entry: flake8
+      types: [python]

.pre-commit-hooks.yaml

@@ -0,0 +1,8 @@
+-   id: black
+    name: black
+    description: 'Black: The uncompromising Python code formatter'
+    entry: black
+    language: python
+    language_version: python3
+    types: [python]
+

README.rst

@@ -0,0 +1,75 @@
+.. image:: https://readthedocs.org/projects/gb/badge/?version=latest
+:target: https://gb.readthedocs.io/en/latest/?badge=latest
+:alt: Documentation Status
+
+gb
+##
+
+`gb` or gopherball is a gopher server written in Python with the main goals of
+ease of use and integration. The name gopherball is inspired by a recurring
+theme in the Calvin & Hobbes comicbooks and a tongue in cheek reference of an
+alternative to the World Wide Web as we know it today.
+
+Examples
+========
+Quick examples to get you running.
+
+`gb --mode=implicit .` will start a gopher server on `127.0.0.1` port `7070` serving
+a recursive index of files starting from the current directory.
+
+`gb --mode=explit /home/user/explicit.json` will start a gopher server on 
+`127.0.0.1` port `7070` with an index generated from the passed configuration
+file. For the format of this file see `gb`'s documentation.
+
+`gb --mode=implicit --magic .` will start `gb` in magic-mode on `127.0.0.1` port
+`7070`. Magic mode will make `gb` parse .txt files as templates. For more
+information on magic mode see `gb`'s documentation.
+
+`gb --mode=implicit --host="127.1.1.1 --port 1025 .` will start `gb` in implicit
+mode on the chosen ip and port. Note that using ports under 1024 requires
+superuser permissions!
+
+Technology
+==========
+`gb` is written with the help of Python 3.5 and higher and the Tornado
+framework for its networking.
+
+Modes
+=====
+`gb` has two main modes of operation that are commonly used. Each has its
+appeal for differing usecases.
+
+implicit
+--------
+Implicit mode serves a directory recursively. Indexes are automatically
+generated and text files are served to the client. Data files are also
+supported.
+
+explicit
+--------
+Explicit mode requires a configuration which maps each path to a certain
+asset. This mode will generate indexes not based on the file system but based
+on a configuration file.
+
+Magic
+=====
+`gb` will serve all non-directories as type 9 files, these are non-readable
+files and most clients will prompt for download. Turning on magic with
+`--magic` will let `gb` try to determine the correct filetypes.
+
+Turning on magic will also start templating special `.gb` files. See
+documentation for what you can do with templating.
+
+Contributing
+============
+The source code for `gb` lives on GitHub where you can also submit issues and
+pull requests. It mostly needs help by people with the ability to test in
+various clients and libraries that might still support the gopher protocol.
+
+Typing
+======
+An often asked question is why `gb` does not use any of Python 3.6+'s type
+annotations. The answer is quite simply that `gb` wants to support `pypy` as
+well as CPython. When `pypy` catches up to 3.6 type annotations will be added.
+
+`gb` loves to run on `pypy` so give it a whirl!

docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'gb'
+copyright = '2018, supakeen'
+author = 'supakeen'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = '1.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'gbdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'gb.tex', 'gb Documentation',
+     'supakeen', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'gb', 'gb Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'gb', 'gb Documentation',
+     author, 'gb', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']

docs/index.rst

@@ -0,0 +1,20 @@
+.. gb documentation master file, created by
+   sphinx-quickstart on Sat Nov 17 20:12:39 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to gb's documentation!
+==============================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

gb/__main__.py

@@ -0,0 +1,4 @@
+from gb.command import main
+
+
+raise SystemExit(main(auto_envvar_prefix="GB"))

gb/command.py

@@ -0,0 +1,85 @@
+import sys
+import ipaddress
+import logging
+
+import click
+import tornado.iostream
+
+import gb.shared
+import gb.server
+
+
+log = logging.getLogger(__name__)
+
+modes = {"implicit": gb.server.ImplicitGopherServer, "explicit": gb.server.ExplicitGopherServer}
+
+
+def bail(message):
+    """Write a message to stderr and exit unsuccesfully."""
+    print(message, file=sys.stderr)
+    raise SystemExit(1)
+
+
+@click.command()
+@click.option(
+    "--mode",
+    "-m",
+    required=True,
+    type=click.Choice(modes.keys()),
+    help="Mode to run as.",
+)
+@click.option(
+    "--host",
+    "-h",
+    default="127.0.0.1",
+    show_default=True,
+    help="IP address to listen on.",
+)
+@click.option(
+    "--port", "-p", default=7070, show_default=True, help="Port to listen on."
+)
+@click.option(
+    "--verbose",
+    "-v",
+    count=True,
+    help="Verbosity, passing more heightens the verbosity.",
+)
+@click.option(
+    "--magic",
+    default=False,
+    help="Enable magic mode which will try to guess filetypes by extension and content.",
+    show_default=True,
+    is_flag=True,
+)
+@click.argument(
+    "path", required=True, type=click.Path(exists=True), envvar="GB_PATH"
+)
+def main(mode, host, port, path, verbose, magic):
+    """`gb` or gopherball is a modern server for the Gopher protocol."""
+
+    if port < 0 or port > 65535:
+        return bail("Invalid port supplied.")
+
+    try:
+        host = ipaddress.ip_address(host)
+    except ValueError:
+        return bail("Unparseable IP supplied.")
+
+    if mode not in modes:
+        return bail("Invalid mode supplied.")
+
+    logging.basicConfig(level=logging.INFO)
+
+    gb.shared.verbosity = verbose
+
+    server = modes[mode](path, magic)
+    server.listen(port)
+    server.start(0)
+
+    tornado.ioloop.IOLoop.current().start()
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(auto_envvar_prefix="GB"))

gb/document.py

@@ -0,0 +1,17 @@
+import gb.protocol
+
+
+class Document:
+    def __init__(self):
+        self.entries = []
+
+    def add_entry(self, entry):
+        self.entries.append(entry)
+
+    def as_bytes(self):
+        chunks = []
+
+        for entry in self.entries:
+            chunks.append(gb.protocol.template.format(e=entry) + gb.protocol.crlf)
+
+        return "".join(chunks)

gb/entry.py

@@ -0,0 +1,82 @@
+import gb.protocol
+
+
+class Entry:
+    host = "localhost"
+    port = "7070"
+
+    def __init__(self, text, selector):
+        self.text = text
+        self.selector = selector
+
+
+class Text(Entry):
+    code = 0
+
+
+class Directory(Entry):
+    code = 1
+
+    def __init__(self, text, selector):
+        self.text = text + "/"
+        self.selector = selector
+
+
+class CCSO(Entry):
+    code = 2
+
+
+class Error(Entry):
+    code = 3
+
+
+class BinHex(Entry):
+    code = 4
+
+
+class DOS(Entry):
+    code = 5
+
+
+class UU(Entry):
+    code = 6
+
+
+class Search(Entry):
+    code = 7
+
+
+class Telnet(Entry):
+    code = 8
+
+
+class Binary(Entry):
+    code = 9
+
+
+class Mirror(Entry):
+    code = "+"
+
+
+class GIF(Entry):
+    code = "g"
+
+
+class Image(Entry):
+    code = "I"
+
+
+class Telnet3270(Entry):
+    code = "T"
+
+
+class HTML(Entry):
+    code = "h"
+
+
+class Information(Entry):
+    code = "i"
+
+
+class Sound(Entry):
+    code = "s"

gb/magic.py

@@ -0,0 +1,14 @@
+import magic
+
+import gb.entry
+
+
+mime_entry = {"text": gb.entry.Text}
+
+
+def guess_type(path):
+    """Guess the type of a path and return the corresponding Entry for that
+       type with its data loaded."""
+
+    guess = magic.Magic(mime=True).from_file(path).split("/")[0]
+    return mime_entry.get(guess, gb.entry.Binary)

gb/mode.py

@@ -0,0 +1,63 @@
+import os
+
+import gb.protocol
+import gb.document
+import gb.entry
+import gb.safe
+import gb.magic
+
+
+class Mode:
+    def __init__(self, base_path, magic):
+        self.base_path = os.path.abspath(base_path)
+        self.magic = magic
+
+
+class ImplicitMode(Mode):
+    def lookup(self, path):
+        """Look up a path within our base path and return the contents in
+           pre-rendered Gopher format!"""
+
+        path = gb.safe.relativize(self.base_path, path)
+
+        if os.path.islink(path):
+            raise ValueError()
+
+        if os.path.isdir(path):
+            return self._directory(path)
+        elif os.path.isfile(path):
+            return self._file(path)
+
+    def _directory(self, path):
+        """Render the files in a directory as gopher data."""
+        response = gb.document.Document()
+
+        for entry in os.listdir(path):
+            entry = os.path.join(path, entry)
+
+            if os.path.isdir(entry):
+                item = gb.entry.Directory
+            elif os.path.isfile(entry):
+                if self.magic:
+                    item = gb.magic.guess_type(entry)
+                else:
+                    item = gb.entry.Binary
+
+            response.add_entry(
+                item(os.path.basename(entry), entry[len(self.base_path) :])
+            )
+
+        return response.as_bytes()
+
+    def _file(self, path):
+        with open(path) as f:
+            return f.read()
+
+
+class ExplicitMode(Mode):
+    def __init__(self, base_path, magic):
+        self.base_path = os.path.abspath(base_path)
+        self.magic = magic
+
+    def lookup(self):
+        raise NotImplementedError

gb/protocol.py

@@ -0,0 +1,28 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+crlf = "\r\n"
+eof = "."
+
+template = "{e.code}{e.text}\t{e.selector}\t{e.host}\t{e.port}"
+
+
+def clean_selector(selector):
+    """Strip the end off of a selector and normalize an empty one."""
+    selector = selector.rstrip(crlf)
+
+    if selector == "":
+        selector = "/"
+
+    return selector
+
+
+def is_valid_selector(selector):
+    """Validate the selector according to gopher rules."""
+
+    # TODO implement <TAB> denoted search
+
+    # These characters are forbidden in gopher selectors and should immediately
+    # terminate the connection
+    return not any(forbidden in selector for forbidden in "\t\r\n")

gb/safe.py

@@ -0,0 +1,24 @@
+import os
+
+
+def relativize(base, path):
+    """Join base and path making sure that the resulting path lies
+       within the base. We do this by making path relative if it's an
+       absolute path then verifying if it's still within the base after
+       joining."""
+
+    base = os.path.abspath(base)
+
+    # The nif the path starts with a separator we remove that
+    if path.startswith(os.path.sep):
+        path = path[len(os.path.sep) :]
+
+    dest = os.path.join(base, path)
+    dest = os.path.abspath(dest)
+
+    if os.path.commonprefix([base, dest]) != base:
+        # TODO we want a better way to communicate this to our server
+        # TODO so it can cleanly close the connection
+        raise Exception("welp")
+
+    return dest

gb/server.py

@@ -0,0 +1,68 @@
+import logging
+
+import tornado.tcpserver
+import tornado.iostream
+
+import gb.protocol
+import gb.mode
+
+
+log = logging.getLogger(__name__)
+
+
+class GopherServer(tornado.tcpserver.TCPServer):
+    async def handle_stream(self, stream, address):
+        log.debug("Accepted connection from %s", address)
+
+        while True:
+            try:
+                # clean this up, split on \n but clean both
+                selector = await stream.read_until(b"\n")  # gb.protocol.crlf)
+                selector = selector.decode("ascii")
+
+                # clean this up, split on \n but clean both with \r
+                selector = gb.protocol.clean_selector(selector)
+
+                # If this is not a valid selector we immediately terminate
+                # the connection
+                if not gb.protocol.is_valid_selector(selector):
+                    log.warn("%s requested invalid selector %r, terminating.", address, selector)
+                    await self.close_stream(stream)
+                    continue
+
+                if not selector:
+                    selector = "/"
+
+                log.info("%s requested %r", address, selector)
+
+                # Use our mode specific lookup to get our response
+                resp = await self.lookup(stream, selector)
+                resp = resp.encode("ascii")
+
+                # Write and exit the connection
+                await stream.write(resp)
+                await self.close_stream(stream)
+            except tornado.iostream.StreamClosedError:
+                log.debug("Lost connection from %s", address)
+                break
+
+    async def close_stream(self, stream):
+        """Gopher connections are closed by writing a . on a single line then
+           closing the underlying transport."""
+        await stream.write(gb.protocol.eof.encode("ascii"))
+        await stream.write(gb.protocol.crlf.encode("ascii"))
+        stream.close()
+
+
+class ImplicitGopherServer(GopherServer):
+    def __init__(self, path, magic):
+        super().__init__()
+        log.info("Starting ImplicitGopherServer with path %s", path)
+        self.mode = gb.mode.ImplicitMode(path, magic)
+
+    async def lookup(self, stream, data):
+        return self.mode.lookup(data)
+
+
+class ExplicitGopherServer(GopherServer):
+    pass

gb/shared.py

@@ -0,0 +1 @@
+verbosity = 0

pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+
+[tool.black]
+line-length = 80
+py36 = false
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.eggs
+  | \.mypy_cache
+  | \.tox
+  | venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+

setup.py

@@ -0,0 +1,17 @@
+from setuptools import setup
+
+
+setup(
+    name="gb",
+    version="1.0.0",
+    description="A Python gopher server.",
+    url="https://github.com/supakeen/gb",
+    author="supakeen",
+    author_email="cmdr@supakeen.com",
+    packages=[],
+    install_requires=["tornado", "click", "python-magic"],
+    entry_points={"console_scripts": ["gb=gb.command:main"]},
+    tests_require=["nose", "aiounittest"],
+    extras_require={"dev": ["pre-commit", "flake8", "black", "nose"]},
+    test_suite="nose.collector",
+)

tests/test_protocol.py

@@ -0,0 +1,13 @@
+import unittest
+
+import gb.protocol
+
+
+class TestProtocolCleanSelector(unittest.TestCase):
+    def test_empty_selector(self):
+        assert "/" == gb.protocol.clean_selector("\r\n")
+        assert "/" == gb.protocol.clean_selector("\n")
+
+    def test_word_selector(self):
+        assert "foo" == gb.protocol.clean_selector("foo\r\n")
+        assert "foo" == gb.protocol.clean_selector("foo\n")

tests/test_safe.py

@@ -0,0 +1,9 @@
+import unittest
+
+import gb.safe
+
+
+class TestSafeRelativize(unittest.TestCase):
+    def test_relative_path(self):
+        with self.assertRaises(Exception):
+            gb.safe.relativize("/tmp", "../../../")