Add or edit site

11 files changed, 459 insertions, 22 deletions

diff --git a/setup.py b/setup.py
index 91a284d..c2aecc9 100644
--- a/setup.py
+++ b/setup.py
@@ -13,10 +13,11 @@ except FileExistsError:
 
 with open("README.md", "r") as f:
     long_description = f.read()
+    f.close()
 
 setuptools.setup(
     name="utab",  # Replace with your own username
-    version="0.1.0",
+    version="0.2.0",
     author="Frederick Yin",
     author_email="fkfd@macaw.me",
     description="Web browser new tab dashboard HTTP daemon",
@@ -24,6 +25,7 @@ setuptools.setup(
     long_description_content_type="text/markdown",
     url="https://git.sr.ht/~fkfd/utab",
     packages=setuptools.find_packages(),
+    install_requires=["flask>=1.1.2", "beautifulsoup4>=4.3.2", "requests>=2.1.0"],
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: BSD License",
diff --git a/utab/__main__.py b/utab/__main__.py
index 5e619ed..df92dbc 100644
--- a/utab/__main__.py
+++ b/utab/__main__.py
@@ -2,9 +2,11 @@ from flask import Flask, Response, request, redirect, abort
 from appdirs import user_data_dir
 from pathlib import Path
 import urllib
+from mimetypes import guess_type
 import sys
 import yaml
 import csv
+from .pyfav import get_favicon_url
 from .rendering import *
 
 app = Flask(__name__)
@@ -13,19 +15,29 @@ app = Flask(__name__)
 # and sites json file at utab/sites.json
 data_dir = user_data_dir(appname="utab")
 template_fp = Path(data_dir) / "index.html"
+site_form_fp = Path(data_dir) / "site.html"
 css_dir = Path(data_dir) / "css"
+icons_dir = Path(data_dir) / "icons"
 try:
-    template = Path.open(template_fp).read()
+    with open(template_fp) as f:
+        template = f.read()
+        f.close()
+    with open(site_form_fp) as f:
+        site_form = f.read()
+        f.close()
 except FileNotFoundError:
-    print("Template file not found.")
+    print("One or more template files are not found.")
     sys.exit(1)
 
+
 sites_fp = Path(data_dir) / "sites.csv"
 config_fp = Path(data_dir) / "config.yml"
 with open(config_fp) as f:
-    config = yaml.load(f.read())
+    config = yaml.load(f.read(), Loader=yaml.FullLoader)
     f.close()
 
+sites_grid_dimensions = {"columns": config["columns"], "rows": config["rows"]}
+
 
 def read_sites():
     with open(sites_fp) as f:
@@ -34,34 +46,128 @@ def read_sites():
         return sites
 
 
+def write_sites(data):
+    with open(sites_fp, "w") as f:
+        csv.writer(f).writerows(data)
+        f.close()
+
+
+def append_site(site):
+    with open(sites_fp, "a") as f:
+        csv.writer(f).writerow(site)
+        f.close()
+
+
 @app.route("/")
 def index():
+    # render sites in a grid.
+    # each cell links to /go/<escaped_url>
     return render_page(
         template,
-        sites=render_sites(
-            read_sites(), columns=config["columns"], rows=config["rows"]
-        ),
+        site_heading="Top Sites",
+        sites=render_sites(read_sites(), **sites_grid_dimensions),
     )
 
 
 @app.route("/go/<path:url>")
 def visit_site(url):
-    print(url)
+    # log this visit, then redirect to the unescaped url
     url_unesc = urllib.parse.unquote(url)  # unescaped url
-    print(url_unesc)
     sites = read_sites()
     for i, s in enumerate(sites):
-        if s[0] == url_unesc:
+        if s[URL] == url_unesc:
             sites[i][VISITS] = str(int(sites[i][VISITS]) + 1)
-            with open(sites_fp, "w") as f:
-                # update visits
-                csv.writer(f).writerows(sites)
-                f.close()
+            write_sites(sites)
             return redirect(url_unesc, 302)
+    return abort(404)
+
+
+@app.route("/new")
+def add_site():
+    # /new => a page asking for url, title, etc.
+    # /new?url=<escaped_url>&title=<title>&... => add this site if it DNE, else abort
+    url, title, favicon = (
+        request.args.get("url"),
+        request.args.get("title"),
+        request.args.get("favicon"),
+    )
+    if url:
+        sites = read_sites()
+        for s in sites:
+            if s[URL] == url:
+                return "A site with the same URL already exists.", 403
+        # now we have ensured there isn't such URL in sites
+        if favicon:
+            favicon_src = favicon
+        else:
+            # get its favicon url
+            # fetch favicon url from the root page (w/o path):
+            url_split = urllib.parse.urlsplit(url)
+            root = url_split.scheme + "://" + url_split.netloc
+            favicon_src = get_favicon_url(root)
+
+        append_site([url, title, favicon_src, 0])
+        return redirect("/", 302)
+    else:  # no request params; let user fill form
+        return render_page(
+            site_form,
+            site_heading="Add site",
+            url="https://",
+            title="",
+            favicon_src="",
+            favicon_placeholder="Leave blank to auto-retrieve favicon. Base64 is allowed.",
+            action="new",
+        )
+
+
+@app.route("/edit/<path:url>")
+def edit_site(url):
+    # /edit/ => /edit
+    # /edit/<escaped_url> => form with original data pre-filled for user to modify
+    # /edit/<escaped_url>?url=<escaped_new_url>&... => edit this site in database
+    if not url:
+        return redirect("/edit", 301)
+    url_unesc = urllib.parse.unquote(url)
+    sites = read_sites()
+    for i, s in enumerate(sites):
+        if s[URL] == url_unesc:
+            new_url, new_title, new_favicon = (
+                request.args.get("url"),
+                request.args.get("title"),
+                request.args.get("favicon"),
+            )
+            if not new_url:
+                return render_page(
+                    site_form,
+                    site_heading="Edit site",
+                    url=s[URL],
+                    title=s[TITLE],
+                    favicon_src=s[FAVICON],
+                    favicon_placeholder="Base64 is allowed.",
+                    action="edit/" + urllib.parse.quote(s[URL], safe=""),
+                )
+            sites[i][URL] = new_url
+            sites[i][TITLE] = new_title
+            sites[i][FAVICON] = new_favicon
+            write_sites(sites)
+            return redirect("/", 302)
+    return abort(404)
+
+
+@app.route("/edit")
+def select_site_to_edit():
+    # render sites in a grid.
+    # each cell links to /edit/<escaped_url>
+    return render_page(
+        template,
+        site_heading="Select site to edit",
+        sites=render_sites(read_sites(), action="edit", **sites_grid_dimensions),
+    )
 
 
 @app.route("/css/<string:filename>")
 def serve_css(filename):
+    # serve static CSS because browsers forbid file:// scheme
     try:
         with open(css_dir / filename) as f:
             resp = Response(f.read(), 200, {"Content-Type": "text/css"})
@@ -73,5 +179,19 @@ def serve_css(filename):
         return abort(500)
 
 
+@app.route("/icons/<string:filename>")
+def serve_icon(filename):
+    try:
+        fp = icons_dir / filename
+        with open(fp, "rb") as f:
+            resp = Response(f.read(), 200, {"Content-Type": guess_type(fp)[0]})
+            f.close()
+            return resp
+    except FileNotFoundError:
+        return abort(404)
+    except:
+        return abort(500)
+
+
 # run on localhost only
 app.run("127.0.0.1", 64366)
diff --git a/utab/data/css/index.css b/utab/data/css/index.css
index 5f03851..651a540 100644
--- a/utab/data/css/index.css
+++ b/utab/data/css/index.css
@@ -20,13 +20,50 @@ body {
   height: 80px;
 }
 
-.sites-item:hover {
-  border: #ccc 2px solid;
-}
-
 .site-favicon {
   width: 64px;
   height: 64px;
   position: relative;
   top: 8px;
 }
+
+.form-input {
+  border: #888 2px solid;
+  width: 40%;
+  height: 2em;
+  background-color: inherit;
+  color: inherit;
+  margin-left: 20px;
+  margin-bottom: 20px;
+}
+
+.form-button {
+  border: #888 2px solid;
+  border-radius: 5px;
+  height: 36px;
+  margin: 20px;
+  padding: 10px;
+  background-color: inherit;
+  color: white;
+}
+
+.sites-item:hover,
+.form-input:hover,
+.form-button:hover {
+  border: #ccc 2px solid;
+}
+
+.ctrl {
+  margin-top: 20px;
+}
+
+.ctrl-icon {
+  width: 48px;
+  height: 48px;
+  margin: 20px;
+  opacity: 60%;
+}
+
+.ctrl-icon:hover {
+  opacity: 100%;
+}
diff --git a/utab/data/icons/home.svg b/utab/data/icons/home.svg
new file mode 100644
index 0000000..a72c187
--- /dev/null
+++ b/utab/data/icons/home.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="8" height="8" viewBox="0 0 8 8">
+  <path style="fill:#ffffff;fill-opacity:1" id="path2" d="M4 0l-4 3h1v4h2v-2h2v2h2v-4.03l1 .03-4-3z" />
+</svg>
diff --git a/utab/data/icons/pencil.svg b/utab/data/icons/pencil.svg
new file mode 100644
index 0000000..e1df5bf
--- /dev/null
+++ b/utab/data/icons/pencil.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="8" height="8" viewBox="0 0 8 8">
+  <path style="fill:#ffffff;fill-opacity:1" id="path2" d="M6 0l-1 1 2 2 1-1-2-2zm-2 2l-4 4v2h2l4-4-2-2z" />
+</svg>
diff --git a/utab/data/icons/plus.svg b/utab/data/icons/plus.svg
new file mode 100644
index 0000000..bfbe2aa
--- /dev/null
+++ b/utab/data/icons/plus.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="8" height="8" viewBox="0 0 8 8">
+  <path style="fill:#ffffff;fill-opacity:1" id="path2" d="M3 0v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2z" />
+</svg>
diff --git a/utab/data/index.html b/utab/data/index.html
index bc6b97b..e2f20b0 100644
--- a/utab/data/index.html
+++ b/utab/data/index.html
@@ -1,11 +1,26 @@
 <!DOCTYPE html>
+<!--Routes using this template:
+/
+/edit
+-->
 <head>
   <title>utab</title>
 </head>
 <body>
-  <h2>Top Sites</h2>
+  <h2>%site_heading%</h2>
   <div id="sites">
     %sites%
   </div>
+  <footer class="ctrl">
+    <a href="/">
+      <img class="ctrl-icon" src="/icons/home.svg" />
+    </a>
+    <a href="/new">
+      <img class="ctrl-icon" src="/icons/plus.svg" />
+    </a>
+    <a href="/edit">
+      <img class="ctrl-icon" src="/icons/pencil.svg" />
+    </a>
+  </footer>
   <link rel="stylesheet" type="text/css" href="/css/index.css" />
 </body>
diff --git a/utab/data/site.html b/utab/data/site.html
new file mode 100644
index 0000000..1406af9
--- /dev/null
+++ b/utab/data/site.html
@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<!--Routes using this template:
+/new/<url>
+/edit/<url>
+-->
+<head>
+  <title>utab - add new</title>
+</head>
+<body>
+  <h2>%site_heading%</h2>
+  <form action="/%action%" method="GET">
+    <label for="url">URL:</label>
+    <input class="form-input" name="url" type="url" value="%url%" autofocus />
+    <br />
+    <label for="title">Title:</label>
+    <input class="form-input" name="title" type="text" value="%title%" />
+    <br />
+    <label for="favicon">Favicon:</label>
+    <input
+      class="form-input"
+      name="favicon"
+      type="url"
+      value="%favicon_src%"
+      placeholder="%favicon_placeholder%"
+    />
+    <br />
+    <input class="form-button" type="submit" />
+    <a href="/"><button class="form-button">Cancel</button></a>
+  </form>
+  <link rel="stylesheet" type="text/css" href="/css/index.css" />
+</body>
diff --git a/utab/pyfav/__init__.py b/utab/pyfav/__init__.py
new file mode 100644
index 0000000..0bd4907
--- /dev/null
+++ b/utab/pyfav/__init__.py
@@ -0,0 +1,3 @@
+from .pyfav import get_favicon_url
+from .pyfav import parse_markup_for_favicon
+from .pyfav import download_favicon
\ No newline at end of file
diff --git a/utab/pyfav/pyfav.py b/utab/pyfav/pyfav.py
new file mode 100644
index 0000000..34b4f93
--- /dev/null
+++ b/utab/pyfav/pyfav.py
@@ -0,0 +1,220 @@
+"""
+MIT License
+Copyright (c) 2013 Matthew Phillips
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the 
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+"""
+This is PyFav. It's a python package that helps you download favicons.
+
+Find the project on GitHub at https://github.com/phillipsm/pyfav
+and in PyPI at http://python.org/pypi/pyfav
+
+
+
+The simplest way to get started is to use the download_favicon function.
+
+To download a favicon for it's as simple as,
+
+============
+from favicon import download_favicon
+
+download_favicon('https://www.python.org/')
+============
+
+If you want to be specific in where that favicon gets written to disk,
+
+============
+favicon_saved_at = download_favicon('https://www.python.org/', \
+    file_prefix='python.org-', target_dir='/tmp/favicon-downloads')
+============
+
+
+If you'd prefer to handle the write to disk piece, use the get_favicon_url
+function by itself,
+============
+favicon_url = get_favicon_url('https://www.python.org/')
+============
+"""
+
+
+import urllib, os.path, string, re
+import requests
+from bs4 import BeautifulSoup
+
+
+# Some hosts don't like the requests default UA. Use this one instead.
+headers = {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) \
+        AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 \
+        Safari/537.36"
+}
+
+
+def download_favicon(url, file_prefix="", target_dir="/tmp"):
+    """
+    Given a URL download the save it to disk
+    
+    Keyword arguments:
+    url -- A string. This is the location of the favicon
+    file_prefix - A string. If you want the downloaded favicon filename to
+        be start with some characters you provide, this is a good way to do it.
+    target_dir -- The location where the favicon will be saved.
+    
+    Returns:
+    The file location of the downloaded favicon. A string.
+    """
+
+    parsed_site_uri = urllib.parse.urlparse(url)
+
+    # Help the user out if they didn't give us a protocol
+    if not parsed_site_uri.scheme:
+        url = "http://" + url
+        parsed_site_uri = urllib.parse.urlparse(url)
+
+    if not parsed_site_uri.scheme or not parsed_site_uri.netloc:
+        raise Exception("Unable to parse URL, %s" % url)
+
+    favicon_url = get_favicon_url(url)
+
+    if not favicon_url:
+        raise Exception("Unable to find favicon for, %s" % url)
+
+    # We finally have a URL for our favicon. Get it.
+    response = requests.get(favicon_url, headers=headers)
+    if response.status_code == requests.codes.ok:
+        # we want to get the the filename from the url without any params
+        parsed_uri = urllib.parse.urlparse(favicon_url)
+        favicon_filepath = parsed_uri.path
+        favicon_path, favicon_filename = os.path.split(favicon_filepath)
+
+    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+
+    sanitized_filename = "".join([x if valid_chars else "" for x in favicon_filename])
+
+    sanitized_filename = os.path.join(target_dir, file_prefix + sanitized_filename)
+
+    with open(sanitized_filename, "wb") as f:
+        for chunk in response.iter_content(chunk_size=1024):
+            if chunk:  # filter out keep-alive new chunks
+                f.write(chunk)
+                f.flush()
+
+    return sanitized_filename
+
+
+def parse_markup_for_favicon(markup, url):
+    """
+    Given markup, parse it for a favicon URL. The favicon URL is adjusted
+    so that it can be retrieved independently. If no favicon is found in the 
+    markup we return None.
+    
+    Keyword arguments:
+    markup -- A string containing the HTML markup.
+    url -- A string containing the URL where the supplied markup can be found.
+    We use this URL in cases where the favicon path in the markup is relative.
+    
+    Retruns:
+    The URL of the favicon. A string. If not found, returns None.
+    """
+
+    parsed_site_uri = urllib.parse.urlparse(url)
+
+    soup = BeautifulSoup(markup)
+
+    # Do we have a link element with the icon?
+    icon_link = soup.find("link", rel="icon")
+    if icon_link and icon_link.has_attr("href"):
+
+        favicon_url = icon_link["href"]
+
+        # Sometimes we get a protocol-relative path
+        if favicon_url.startswith("//"):
+            parsed_uri = urllib.parse.urlparse(url)
+            favicon_url = parsed_uri.scheme + ":" + favicon_url
+
+        # An absolute path relative to the domain
+        elif favicon_url.startswith("/"):
+            favicon_url = (
+                parsed_site_uri.scheme + "://" + parsed_site_uri.netloc + favicon_url
+            )
+
+        elif re.match("^data:\w+/\w+;base64,[A-Za-z0-9=/]+", favicon_url):
+            pass  # base64; return verbatim
+
+        # A relative path favicon
+        elif not favicon_url.startswith("http"):
+            path, filename = os.path.split(parsed_site_uri.path)
+            favicon_url = (
+                parsed_site_uri.scheme
+                + "://"
+                + parsed_site_uri.netloc
+                + "/"
+                + os.path.join(path, favicon_url)
+            )
+
+        # We found a favicon in the markup and we've formatted the URL
+        # so that it can be loaded independently of the rest of the page
+        return favicon_url
+
+    # No favicon in the markup
+    return None
+
+
+def get_favicon_url(url):
+    """
+    Returns a favicon URL for the URL passed in. We look in the markup returned
+    from the URL first and if we don't find a favicon there, we look for the 
+    default location, e.g., http://example.com/favicon.ico . We retrurn None if
+    unable to find the file.
+    
+    Keyword arguments:
+    url -- A string. This is the URL that we'll find a favicon for.
+    
+    Returns:
+    The URL of the favicon. A string. If not found, returns None.
+    """
+
+    parsed_site_uri = urllib.parse.urlparse(url)
+
+    # Get the markup
+    try:
+        response = requests.get(url, headers=headers)
+    except:
+        raise Exception("Unable to find URL. Is it valid? %s" % url)
+
+    if response.status_code == requests.codes.ok:
+        favicon_url = parse_markup_for_favicon(response.content, url)
+
+        # We found a favicon in our markup. Return the URL
+        if favicon_url:
+            return favicon_url
+
+    # The favicon doesn't appear to be in the makrup
+    # Let's look at the common locaiton, url/favicon.ico
+    favicon_url = "{uri.scheme}://{uri.netloc}/favicon.ico".format(uri=parsed_site_uri)
+
+    response = requests.get(favicon_url, headers=headers)
+    if response.status_code == requests.codes.ok:
+        return favicon_url
+
+    # No favicon in the markup or at url/favicon.ico
+    return None
diff --git a/utab/rendering.py b/utab/rendering.py
index 731524c..12507f2 100644
--- a/utab/rendering.py
+++ b/utab/rendering.py
@@ -9,7 +9,7 @@ def render_page(template: str, **kwargs):
     return page
 
 
-def render_sites(sites: list, columns=8, rows=4):
+def render_sites(sites: list, columns=8, rows=4, action="go"):
     top_sites = sorted(sites, key=lambda s: int(s[VISITS]), reverse=True)[
         : (columns * rows)  # top col*row sites, default=32
     ]
@@ -25,8 +25,8 @@ def render_sites(sites: list, columns=8, rows=4):
             if col is not None:
                 html += (
                     '<div class="sites-item">'
-                    f'<a class="site" href="/go/{urllib.parse.quote(col[URL], safe="")}">'
-                    f'<img class="site-favicon" src="{col[FAVICON]}" /></a>'
+                    f'<a class="site" href="/{action}/{urllib.parse.quote(col[URL], safe="")}">'
+                    f'<img class="site-favicon" src="{col[FAVICON]}" title="{col[URL]}"/></a>'
                     + f"<p>{col[TITLE]}</p>"
                     + "</div>"
                 )