◊(Local Yarn Code "Check-in [43a06b90]")

*compiled/*
*.woff*
x-*/*
*.*~
web-extra/martin.css
scribbled/*
code-docs/*.css
code-docs/*.js
*/images/*
*.db
*.sqlite
*.pdf
*.ltx
*.html
*.out
*.ltx
*.aux
*.log
*.xml
*.toc
*.mark

#lang racket/base

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

(require deta
         db/base
         db/sqlite3
         threading
         pollen/setup
         racket/match
         "dust.rkt"
         (except-in pollen/core select))

(provide init-cache-db!
         cache-conn                     ; The most eligible bachelor in Neo Yokyo
         (schema-out cache:article)
         (schema-out cache:note)
         (schema-out cache:index-entry)
         (schema-out listing)
         delete-article!
         delete-notes!
         delete-index-entries!
         save-cache-things!
         articles
         articles+notes
         listing-htmls
         fenced-listing
         unfence)

;; Cache DB and Schemas

(define DBFILE (build-path (current-project-root) "vitreous.sqlite"))
(define cache-conn (make-parameter (sqlite3-connect #:database DBFILE #:mode 'create)))

(define-schema cache:article #:table "articles"
  ([id                   id/f #:primary-key #:auto-increment]
   [page                 symbol/f]
   [title-plain          string/f  #:nullable]
   [title-html-flow      string/f  #:nullable]
   [title-specified?     boolean/f #:nullable]
   [published            string/f  #:nullable]
   [updated              string/f  #:nullable]
   [author               string/f  #:nullable]
   [conceal              string/f]
   [series-page          symbol/f  #:nullable]
   [noun-singular        string/f  #:nullable]
   [note-count           integer/f #:nullable]
   [content-html         string/f  #:nullable]
   [disposition          string/f  #:nullable]
   [disp-html-anchor     string/f  #:nullable]
   [listing-full-html    string/f  #:nullable]   ; full content but without notes
   [listing-excerpt-html string/f  #:nullable]   ; Not used for now
   [listing-short-html   string/f  #:nullable])) ; Date and title only

(define-schema cache:note #:table "notes"
  ([id                   id/f #:primary-key #:auto-increment]
   [page                 symbol/f]
   [html-anchor          string/f]
   [title-html-flow      string/f] ; No block-level HTML elements
   [title-plain          string/f]
   [author               string/f]
   [author-url           string/f]
   [published            string/f]
   [disposition          string/f]
   [content-html         string/f]
   [series-page          symbol/f]
   [conceal              string/f]
   [listing-full-html    string/f]
   [listing-excerpt-html string/f]   ; Not used for now
   [listing-short-html   string/f])) ; Date and title only

(define-schema cache:index-entry #:table "index_entries"
  ([id          id/f #:primary-key #:auto-increment]
   [entry       string/f]
   [subentry    string/f]
   [page        symbol/f]
   [html-anchor string/f]))

(define-schema listing
  #:virtual
  ([path        string/f]
   [title       string/f]
   [author      string/f]
   [published   string/f]
   [updated     string/f]
   [html        string/f]))

(define (init-cache-db!)
  (create-table! (cache-conn) 'cache:article)
  (create-table! (cache-conn) 'cache:note)
  (create-table! (cache-conn) 'cache:index-entry))

(define (delete-article! page)
  (query-exec (cache-conn)
              (~> (from cache:article #:as a)
                  (where (= a.page ,(format "~a" page)))
                  delete)))

(define (delete-notes! page)
  (query-exec (cache-conn)
              (~> (from cache:note #:as n)
                  (where (= n.page ,(format "~a" page)))
                  delete)))

(define (delete-index-entries! page)
  (query-exec (cache-conn)
              (~> (from cache:index-entry #:as e)
                  (where (= e.page ,(format "~a" page)))
                  delete)))

(define (save-cache-things! es)
  (void (apply insert! (cache-conn) es)))

;;
;;  ~~~ Fetching articles and notes ~~~
;;

;; (Private use) Conveniece function for the WHERE `series-page` clause
(define (where-series q s)
  (define (s->p x) (format "~a/~a.html" series-folder x))
  (match s
    [(list series ...)
     (where q (in a.series-page ,(map s->p series)))] ; WHERE series-page IN (item1 ...)
    [(or (? string? series) (? symbol? series))
     (where q (= a.series-page ,(s->p series)))]      ; WHERE series-page = "item"
    [#t
     (where q (like a.series-page ,(format "%~a" (here-output-path))))]
    [_ q]))

;; (Private use) Convenience for the WHERE `conceal` NOT LIKE clause
(define (where-not-concealed q)
  (define base-clause (where q (not (like a.conceal "%all%"))))
  (match (listing-context)
    ["" base-clause]
    [(var context) (where base-clause (not (like a.conceal ,(format "%~a%" context))))]))

;; Needed to "parameterize" column names
;; see https://github.com/Bogdanp/deta/issues/14#issuecomment-573344928
(require (prefix-in ast: deta/private/ast))

;; Builds a query to fetch articles
(define (articles type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])
  (define html-field
    (match type
      ['content "content_html"]
      [_ (format "listing_~a_html" type)]))
  (~> (from cache:article #:as a)
      (select (as a.page path)
              (as a.title-plain title)
              a.author
              a.published
              a.updated
              (fragment (ast:as (ast:qualified "a" html-field) "html")))
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

;; Builds a query that returns articles and notes intermingled chronologically
(define (articles+notes type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])
  (define html-field
    (match type
      ['content "content_html"]
      [_ (format "listing_~a_html" type)]))
  (~> (from (subquery
             (~> (from cache:article #:as A)
                 (select 
                  (as A.page path)
                  (as A.title-plain title)
                  A.author
                  A.published
                  A.updated
                  (fragment (ast:as (ast:qualified "A" html-field) "html"))
                  A.series-page
                  A.conceal)
                 (union
                  (~> (from cache:note #:as N)
                      (select 
                       (as (array-concat N.page "#" N.html-anchor) path)
                       (as N.title-plain title)
                       N.author
                       N.published
                       (as "" updated) 
                       (fragment (ast:as (ast:qualified "N" html-field) "html"))
                       N.series-page
                       N.conceal)))))
            #:as a)
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

;; Get all the a list of the HTML all the results in a query
(define (listing-htmls list-query)
  (for/list ([l (in-entities (cache-conn) list-query)])
    (listing-html l)))

;; Return cached HTML of articles and/or notes, fenced within a style txexpr to prevent it being
;; escaped by ->html. See also: definition of `unfence`
(define (fenced-listing q)
  `(style ,@(listing-htmls q)))

;; Remove "<style>" and "</style>" introduced by using ->html on docs containing output from
;; listing functions
(define (unfence html-str)
  (regexp-replace* #px"<[\\/]{0,1}style>" html-str ""))

.fileblock .SCodeFlow {
    padding-top: 0.7em;
    margin-top: 0;
}

.fileblock {
    width: 90%;
}

.fileblock_filetitle{
    background: #eee;
    text-align:right;
    padding: 0.15em;
    border: 1px dotted black;
    border-bottom: none;
}

.terminal, .browser {
    margin-bottom: 1em;
    padding: 0.5em;
    width: 88%;
    background: #fcfcfc;
    color: rgb(150, 35, 105);
}

.terminal .SIntrapara, .browser .SIntrapara, .fileblock .SIntrapara {
    margin: 0 0 0 0;
}

<div class='fossil-doc' data-title='Code Documentation' >
<!-- SPDX-License-Identifier: BlueOak-1.0.0
     This file is licensed under the Blue Oak Model License 1.0.0.
-->
    <div class='iframe-surround'>
        <iframe id='scribble' src="index.html" class="embedded-docs">
        </iframe>
    </div>
</div>

    <script>
        document.getElementById('scribble').src = "index.html?n=" + new Date()/1;
    </script>

#lang racket/base

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

(require deta
         db/base
         threading
         racket/match
         racket/string
         txexpr
         pollen/template
         pollen/decode
         (except-in pollen/core select) ; avoid conflict with deta
         )

(require "dust.rkt" "cache.rkt" "snippets-html.rkt")

(provide parse-and-cache-article!
         cache-index-entries-only!)

(define current-title       (make-parameter #f))
(define current-excerpt     (make-parameter #f))
(define current-notes       (make-parameter '()))
(define current-disposition (make-parameter ""))
(define current-disp-id     (make-parameter ""))

(define (filter-special-tags tx)
  (match (get-tag tx)
    ['title (current-title tx) ""]
    ['excerpt (current-excerpt tx) ""]
    ['excerpt* (current-excerpt tx) `(@ ,@(get-elements tx))] ; splice contents back in
    ['note
     (define note-id (build-note-id tx))
     (cond [(attrs-have-key? tx 'disposition)
            (current-disp-id note-id)
            (current-disposition (attr-ref tx 'disposition))])
     (current-notes (cons (attr-set tx 'note-id note-id) (current-notes))) ""]
    [_ tx]))

;; Save an article and its notes (if any) to the database, and return 
;; (values plain-title [rendered HTML of the complete article])
(define (parse-and-cache-article! pagenode doc)
  (define body-txpr (decode doc #:txexpr-proc filter-special-tags))
  (current-notes (reverse (current-notes)))
  (let* ([pubdate (select-from-metas 'published (current-metas))]
         [doc-html    (->html body-txpr #:splice? #t)]
         [title-specified? (if (current-title) #t #f)]
         [title-val   (or (current-title) (check-for-poem-title doc))]
         [title-tx    (make-article-title pagenode
                                          title-val
                                          body-txpr
                                          (current-disposition)
                                          (current-disp-id))]
         [title-html  (->html title-tx #:splice? #t)]
         [title-plain (tx-strs title-tx)]
         [header      (html$-article-open pagenode title-specified? title-tx pubdate)]
         [series-node (current-series-pagenode)]
         [footertext  (make-article-footertext pagenode
                                               series-node
                                               (current-disposition)
                                               (current-disp-id)
                                               (length (current-notes)))]
         [footer (html$-article-close footertext)]
         [listing-short (html$-article-listing-short pagenode pubdate title-html)]
         [listing-full (string-append header doc-html footer)]
         [listing-excerpt (match (current-excerpt)
                            [#f listing-full]
                            [(var e) (string-append header (html$-article-excerpt pagenode e) footer)])]
         [notes (extract-notes pagenode title-plain (current-notes))]
         [notes-section-html (html$-notes-section (map cadr notes))])
    (thread
     (lambda ()
       (call-with-transaction
        (cache-conn)
        (lambda ()
          (cache-index-entries! pagenode doc) ; note original doc is used here
          (query-exec (cache-conn)
                      (delete (~> (from cache:note #:as n)
                                  (where (= n.page ,(symbol->string pagenode))))))
          (apply insert! (cache-conn) (map car notes))
          (delete-article! pagenode)
          (insert-one! (cache-conn)
                       (make-cache:article
                        #:page pagenode
                        #:title-plain title-plain
                        #:title-html-flow title-html
                        #:title-specified? title-specified?
                        #:published pubdate
                        #:updated (maybe-meta 'updated)
                        #:author (maybe-meta 'author default-authorname)
                        #:conceal (maybe-meta 'conceal)
                        #:series-page series-node
                        #:noun-singular (maybe-meta 'noun (current-series-noun))
                        #:note-count (length (current-notes))
                        #:content-html doc-html
                        #:disposition (current-disposition)
                        #:disp-html-anchor (current-disp-id)
                        #:listing-full-html listing-full
                        #:listing-excerpt-html listing-excerpt
                        #:listing-short-html listing-short))))))
    (values title-plain (string-append header doc-html notes-section-html footer))))

(define (check-for-poem-title doc-txpr)
  (match (car (get-elements doc-txpr))
    [(txexpr 'div
             (list (list 'class "poem"))
             (list* (txexpr 'p
                            (list (list 'class "verse-heading"))
                            heading-elems)
                    _))
     `(title (span [[class "smallcaps"]] "‘" ,@heading-elems "’"))]
    [_ '()]))

;; Return a title txexpr for the current article, constructing a default if no title text was specified.
(define (make-article-title pagenode supplied-title body-tx disposition disp-note-id)
  (define title-elems
    (cond [(null? supplied-title) (list (default-title (get-elements body-tx)))]
          [else (get-elements supplied-title)]))
  
  (define disposition-part
    (cond [(non-empty-string? disposition)
           (define-values (mark _) (disposition-values disposition))
           `(a [[class "disposition-mark"] 
                [href ,(format "~a~a#~a" web-root pagenode disp-note-id)]] 
               ,mark)]
          [else ""]))
  ;; Returns a txexpr, the tag will be discarded by the template/snippets
  `(title ,@title-elems ,disposition-part))

;; Convert a bunch of information about an article into some nice English and links.
(define (make-article-footertext pagenode series disposition disp-note-id note-count)
  (define series-part
    (match (current-series-title)
      [(? non-empty-string? s-title)
       (format "<span class=\"series-part\">This is ~a, part of <a href=\"/~a\">‘~a’</a>.</span>"
               (current-series-noun)
               series
               s-title)]
      [_ ""]))
  (define disp-part
    (cond [(non-empty-string? disposition)
           (define-values (mark verb) (disposition-values disposition))
           (format "Now considered <a href=\"/~a#~a\">~a</a>."
                   pagenode
                   disp-note-id
                   verb)]
          [else ""]))
  (define notes-part
    (cond [(note-count . > . 1)
           (format "There are <a href=\"/~a#furthernotes\">~a notes</a> appended."
                   pagenode
                   note-count)]
          [(and (note-count . > . 0) (string=? disposition ""))
           (format "There is <a href=\"/~a#furthernotes\">a note</a> appended."
                   pagenode)]
          [else ""]))
  
  (cond [(ormap non-empty-string? (list series-part disp-part notes-part))
         (string-join (list series-part disp-part notes-part))]
        [else ""]))

;; ~~~ Notes ~~~

(define (extract-notes pagenode parent-title note-txprs)
  (for/list ([n (in-list note-txprs)])
    (make-note n pagenode parent-title)))

;; Save an individual note to the DB and return the HTML of the complete note as
;; it should appear on an individual article page
(define (make-note note-tx pagenode parent-title-plain)
  (define-values (_ attrs elems) (txexpr->values note-tx))
  (define disposition-attr (maybe-attr 'disposition attrs))
  (define note-date (maybe-attr 'date attrs))
  
  ;; Check required attributes
  (unless (non-empty-string? note-date)
    (raise-arguments-error 'note "required attr missing: date" "attrs" attrs))
  (unless (or (string=? "" disposition-attr)
              (>= (length (string-split disposition-attr)) 2))
    (raise-arguments-error 'note
                           "must be in format \"[symbol] [past-tense-verb]\""
                           "disposition attr"
                           disposition-attr))
  (define-values (disp-mark disp-verb) (disposition-values disposition-attr))
  (let* ([note-id (build-note-id note-tx)]
         [title-tx (make-note-title pagenode parent-title-plain)]
         [title-html (->html title-tx #:splice? #t)]
         [author (maybe-attr 'author attrs default-authorname)]
         [author-url (maybe-attr 'author-url attrs)]
         [note-srcline (maybe-attr 'srcline attrs)]
         [content-html (html$-note-contents disp-mark disp-verb elems)]
         [listing-full (html$-note-listing-full pagenode
                                                note-id
                                                title-html
                                                note-date
                                                note-srcline
                                                content-html
                                                author
                                                author-url)])
    (list
     (make-cache:note
      #:page pagenode
      #:html-anchor note-id
      #:title-html-flow title-html
      #:title-plain (tx-strs title-tx)
      #:published note-date
      #:author author
      #:author-url author-url
      #:disposition disposition-attr
      #:series-page (current-series-pagenode)
      #:conceal (or (maybe-attr 'conceal attrs #f) (maybe-meta 'conceal))
      #:content-html content-html
      #:listing-full-html listing-full
      #:listing-excerpt-html listing-full
      #:listing-short-html "")
     (html$-note-in-article note-id note-date content-html author author-url))))

(define (make-note-title pagenode parent-title-plain)
  `(note-title "Re: " (a [[class "cross-reference"]
                          [href ,(format "~a~a" web-root pagenode)]]
                         ,parent-title-plain)))

;; ~~~ Keyword Index Entries ~~~

;; (private) Convert an entry key into a list of at most two elements,
;; a main entry and a sub-entry.
;;   "entry" → '("entry" "")
;;   "entry!sub" → '("entry" "sub")
;;   "entry!sub!why?!? '("entry" "sub")
(define (split-entry str)
  (define splits (string-split str "!"))
  (list (car splits)
        (cadr (append splits (list "")))))

(define (index-entry-txpr? tx)
  (and (txexpr? tx)
       (string=? "index-link" (attr-ref tx 'class "")) ; see definition of html-index
       (attr-ref tx 'data-index-entry #f)))

(define (txexpr->index-entry tx pagenode)
  (match (split-entry (attr-ref tx 'data-index-entry))
    [(list main sub)
     (make-cache:index-entry
      #:entry main
      #:subentry sub
      #:page pagenode
      #:html-anchor (attr-ref tx 'id))]))

;; Get index entries out of metas
(define (current-metas-keyword-entries pagenode)
  (for/list ([kw (in-list (string-split (maybe-meta 'keywords "") #px";\\s*"))])
    (match (split-entry kw)
      [(list main sub)
       (make-cache:index-entry
        #:entry main
        #:subentry sub
        #:page pagenode
        #:html-anchor "")])))

;; Save any index entries in doc to the SQLite cache.
;; Sub-entries are specified by "!" in the index key
(define (cache-index-entries! pagenode doc)
  (define-values (_ entry-txs) (splitf-txexpr doc index-entry-txpr?))
  (define all-entries
    (append (for/list ([etx (in-list entry-txs)]) (txexpr->index-entry etx pagenode))
            (current-metas-keyword-entries pagenode)))
  
  (delete-index-entries! pagenode)
  (save-cache-things! all-entries))

(define (cache-index-entries-only! title pagenode doc)
  (void
   (thread
    (lambda ()
      (call-with-transaction
       (cache-conn)
       (lambda ()
         (cache-index-entries! pagenode doc)
         (delete-article! pagenode)
         (insert-one! (cache-conn)
                       (make-cache:article
                        #:title-plain title
                        #:conceal "blog,feed"
                        #:page pagenode))))))))

#lang racket/base

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

;; Generates an Atom feed from the SQLite cache

(require txexpr
         deta
         racket/match
         racket/file
         racket/date
         racket/string
         racket/sequence
         "dust.rkt"
         "cache.rkt")

(provide main)

(define feed-author default-authorname)
(define feed-author-email "joel@jdueck.net")
(define feed-title "The Local Yarn (Beta)")
(define feed-site-url "https://thelocalyarn.com")
(define feed-item-limit 50)

(define (as-cdata string)
  (cdata #f #f (format "<![CDATA[~a]]>" string))) ; cdata from xml package via txexpr

(define (email-encode str)
  (map char->integer (string->list str)))

;; Atom feeds require dates to be in RFC 3339 format
;; Our published/updated dates only give year-month-day. No need to bother about time zones or DST.
;; So, arbitrarily, everything happens at a quarter of noon UTC.
(define (ymd->rfc3339 datestr)
  (format "~aT11:45:00Z" datestr))

;; For the feed "updated" value, we do want a complete timestamp.
(define (current-rfc3339)
  ;; #f argument to seconds->date forces a UTC timestamp
  (define now (seconds->date (* 0.001 (current-inexact-milliseconds)) #f))
  (define timestamp
    (parameterize [(date-display-format 'iso-8601)]
      (date->string now #t)))
  (string-append timestamp "Z"))

;; Get the data out of the SQLite cache as vectors
(define (fetch-rows)
  (sequence->list
   (in-entities (cache-conn)
                (articles+notes 'content #:series #f #:limit feed-item-limit))))

(define (listing->rss-item lst)
  (match-define (listing _ path title author published updated html) lst)
  (define entry-url (string-append feed-site-url web-root path))
  (define updated-ts (if (non-empty-string? updated) updated published))
  
  `(entry (author (name ,author))
          (published    ,(ymd->rfc3339 published))
          (updated      ,(ymd->rfc3339 updated-ts))
          (title        ,title)
          (link [[rel "alternate"] [href ,entry-url]])
          (id           ,entry-url)
          (summary [[type "html"]]
                   ,(as-cdata html))))

(define (rss-feed)
  (define feed-xpr
    `(feed [[xml:lang "en-us"] [xmlns "http://www.w3.org/2005/Atom"]]
           (title ,feed-title)
           (link [[rel "self"] [href ,(string-append feed-site-url web-root "feed.xml")]])
           (generator [[uri "http://pollenpub.com/"]] "Pollen")
           (id ,(string-append feed-site-url web-root))
           (updated ,(current-rfc3339))
           (author
            (name ,feed-author)
            (email ,@(email-encode feed-author-email)))
           ,@(map listing->rss-item (fetch-rows))))
  (string-append "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
                 (xexpr->string feed-xpr)))

(define (main)
  (display-to-file (rss-feed) "feed.xml" #:mode 'text #:exists 'replace))

#lang racket/base

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

;; Provides fast metadata for series.
;; TO MAKE A NEW SERIES:
;;   1. Create series/my-key.poly.pm
;;   2. Add an entry for my-key to series-list below
;;   3. Use ◊define-meta[series "my-key"] in articles to add them to the series.
;;   4. If ptree-ordered is #t, also create series/my-key.ptree

(require racket/list)
(struct series (key title noun-plural noun-singular ptree-ordered?))

(define series-list
  (make-immutable-hash
   (list
   ;; ------- DEFINE SERIES HERE -----------
            ; Key               Title              plural noun    singular noun phrase
   (+series "marquee-fiction"   "Marquee Fiction" "Inventions"    "an invention"         #f)
   (+series "local-yarn"        "Local Yarn Site Notes" "Project notes" "a project note" #f)
   )))

(define (series-grouped-list)
  (group-by series-noun-plural (hash-values series-list)))

;; Quick macro to save a little typing
(define-syntax-rule (+series key title plural singular ptree)
  (cons key (series key title plural singular ptree)))

(provide (struct-out series)
         series-list
         series-grouped-list)

#lang racket/base

(provide targets)

(define targets '(html))

#lang racket/base

(require racket/file
         pollen/setup
         "../cache.rkt"
         "../snippets-html.rkt")

(provide main)

(define (main)
  (init-cache-db!)
  
  (display-to-file (html$-page-footer)
                   (build-path (current-project-root) "scribbled" "site-footer.html")
                   #:exists 'replace))