◊(Local Yarn Code "Check-in [d3755213]")

Overview
Comment:Build up listing schema, use for RSS feed ([2f082139])
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: d37552133761dc80fd424f07d8d82c03f335eb6f52933a51c3de14842ccb1dfa
User & Date: joel on 2020-02-21 22:43:34
Other Links: manifest | tags
Context
2020-02-22
00:04
Minor code doc edits check-in: e538e325 user: joel tags: trunk
2020-02-21
22:43
Build up listing schema, use for RSS feed ([2f082139]) check-in: d3755213 user: joel tags: trunk
2020-02-20
08:30
Update code docs to reflect [47d9eea54] check-in: b223c8a5 user: joel tags: trunk
Changes

Modified cache.rkt from [fa204a09] to [eeff2fe2].

45
46
47
48
49
50
51
52

53
54
55
56
57
58
59
45
46
47
48
49
50
51

52
53
54
55
56
57
58
59







-
+







   [published            string/f]
   [updated              string/f]
   [author               string/f]
   [conceal              string/f]
   [series-page          symbol/f]
   [noun-singular        string/f]
   [note-count           integer/f]
   [doc-html             string/f]
   [content-html             string/f]
   [disposition          string/f]
   [disp-html-anchor     string/f]
   [listing-full-html    string/f]   ; full content but without notes
   [listing-excerpt-html string/f]   ; Not used for now
   [listing-short-html   string/f])) ; Date and title only

(define-schema cache:note #:table "notes"
86
87
88
89
90
91
92
93
94
95






96
97
98
99
100
101
102
86
87
88
89
90
91
92



93
94
95
96
97
98
99
100
101
102
103
104
105







-
-
-
+
+
+
+
+
+







   [entry       string/f]
   [subentry    string/f]
   [page        symbol/f]
   [html-anchor string/f]))

(define-schema listing
  #:virtual
  ([html        string/f]
   [published   date/f]
   [series-page symbol/f]))
  ([path        string/f]
   [title       string/f]
   [author      string/f]
   [published   string/f]
   [updated     string/f]
   [html        string/f]))

(define (init-cache-db!)
  (create-table! (cache-conn) 'cache:article)
  (create-table! (cache-conn) 'cache:note)
  (create-table! (cache-conn) 'cache:series)
  (create-table! (cache-conn) 'cache:index-entry))

137
138
139
140
141
142
143
144




145
146



147
148
149


150
151
152
153
154
155
156
157
158




159
160
161
162
163
164









165
166
167
168
169
170









171
172
173
174
175
176
177
140
141
142
143
144
145
146

147
148
149
150
151

152
153
154
155


156
157
158
159
160
161
162
163
164
165

166
167
168
169
170
171




172
173
174
175
176
177
178
179
180
181
182




183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198







-
+
+
+
+

-
+
+
+

-
-
+
+








-
+
+
+
+


-
-
-
-
+
+
+
+
+
+
+
+
+


-
-
-
-
+
+
+
+
+
+
+
+
+








;; Needed to "parameterize" column names
;; see https://github.com/Bogdanp/deta/issues/14#issuecomment-573344928
(require (prefix-in ast: deta/private/ast))

;; Builds a query to fetch articles
(define (articles type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])
  (define html-field (format "listing_~a_html" type))
  (define html-field
    (match type
      ['content "content_html"]
      [_ (format "listing_~a_html" type)]))
  (~> (from cache:article #:as a)
      (select (fragment (ast:as (ast:qualified "a" html-field) "html"))
      (select (as a.page path)
              (as a.title-plain title)
              a.author
              a.published
              a.series-page
              a.conceal)
              a.updated
              (fragment (ast:as (ast:qualified "a" html-field) "html")))
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

;; Builds a query that returns articles and notes intermingled chronologically
(define (articles+notes type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])
  (define html-field (format "listing_~a_html" type))
  (define html-field
    (match type
      ['content "content_html"]
      [_ (format "listing_~a_html" type)]))
  (~> (from (subquery
             (~> (from cache:article #:as A)
                 (select (fragment (ast:as (ast:qualified "A" html-field) "html"))
                         A.published
                         A.series-page
                         A.conceal)
                 (select 
                  (as A.page path)
                  (as A.title-plain title)
                  A.author
                  A.published
                  A.updated
                  (fragment (ast:as (ast:qualified "A" html-field) "html"))
                  A.series-page
                  A.conceal)
                 (union
                  (~> (from cache:note #:as N)
                      (select (fragment (ast:as (ast:qualified "N" html-field) "html"))
                              N.published
                              N.series-page
                              N.conceal)))))
                      (select 
                       (as (array-concat N.page "#" N.html-anchor) path)
                       (as N.title-plain title)
                       N.author
                       N.published
                       (as "" updated) 
                       (fragment (ast:as (ast:qualified "N" html-field) "html"))
                       N.series-page
                       N.conceal)))))
            #:as a)
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

Modified code-docs/cache.scrbl from [bc3bc40e] to [1876eb41].

88
89
90
91
92
93
94
95

96
97
98
99

100
101
102
103
104
105
106
107
108







109
110
111
112
113
114
115
116
117
118
119
120
121
122





123
124
125
126
127
128
129
88
89
90
91
92
93
94

95
96
97
98

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125




126
127
128
129
130
131
132
133
134
135
136
137







-
+



-
+









+
+
+
+
+
+
+










-
-
-
-
+
+
+
+
+







Returns the HTML bodies for the articles and/or notes returned by @racket[_listing-query] as a list
of strings. The @racket[_query] will usually be the result of a call to @racket[articles] or
@racket[articles+notes], but can be any custom query that projects onto the @racket[listing] schema
(see @racket[project-onto]).

}

@deftogether[(@defproc[(articles [type (or/c 'full 'excerpt 'short)]
@deftogether[(@defproc[(articles [type (or/c 'full 'excerpt 'short 'content)]
                                 [#:series series (or/c string? (listof string?) boolean?) #t]
                                 [#:limit limit integer? -1]
                                 [order stringish? 'desc]) query?]
              @defproc[(articles+notes [type (or/c 'full 'excerpt 'short)]
              @defproc[(articles+notes [type (or/c 'full 'excerpt 'short 'content)]
                                       [#:series series (or/c string? (listof string?) boolean?) #t]
                                       [#:limit limit integer? -1]
                                       [order stringish? 'desc]) query?])]{

Create a query that fetches either articles only, or articles and notes intermingled, respectively.
The results will be sorted by publish date according to @racket[_order] and optionally limited to
a particular series. Use the resulting query with the @racket[listing-htmls] or
@racket[fenced-listing] functions provided by this module, or with deta’s @racket[in-entities] if
you want to work with the @racket[listing] schema structs.

The @racket[_type] parameter specifies what version of the articles’ and notes’ HTML markup you
want. For HTML suitable for listing several articles and/or notes together on the same page, use
@racket['full] (the full content but not including @tech{notes}), @racket['excerpt] (like full but
abbreviated to only the excerpt if one was specified) or @racket['short] (date and title only). Use
@racket['content] to get the entire HTML content, including any notes but not including any header
or footer. (This is the option used in the RSS feed.)

If @racket[_series] expression evaluates to @racket[#f], articles will not be filtered by series. If
it evaluates to @racket[#t] (the default), articles will be filtered by those that specify the
current output of @racket[here-output-path] in their @tt{series_pagenode} column in the SQLite
cache. If a string or a symbol is supplied, articles will be filtered by those containing the result
of @racket[(format "series/~a.html" _series)] in their @tt{series_pagenode} column in the SQLite
cache. If a list of strings or symbols is provided, this @racket[format] operation will be applied
to each of its members and articles whose @tt{series_pagenode} column matches any of the resulting
values will be included.

The @racket[_order] expression must evaluate to either @racket["ASC"] or @racket["DESC"] and the
@racket[_limit] expressions must evaluate to a value suitable for use in the @tt{LIMIT} clause of
@ext-link["https://sqlite.org/lang_select.html"]{a SQLite @tt{SELECT} statement}. An expression that
evaluates to a negative integer (the default) is the same as having no limit.
The @racket[_order] expression must evaluate to either @racket["ASC"] or @racket["DESC"] (or
equivalent symbols) and the @racket[_limit] expressions must evaluate to a value suitable for use in
the @tt{LIMIT} clause of @ext-link["https://sqlite.org/lang_select.html"]{a SQLite @tt{SELECT}
statement}. An expression that evaluates to a negative integer (the default) is the same as having
no limit.

Typically you will pass these functions by name to listing functions like @racket[fenced-listing]
rather than calling them directly.

@examples[#:eval example-eval
(articles 'full)]
}
170
171
172
173
174
175
176
177

178
179
180
181
182
183
184
178
179
180
181
182
183
184

185
186
187
188
189
190
191
192







-
+







                           [published            string/f]
                           [updated              string/f]
                           [author               string/f]
                           [conceal              string/f]
                           [series-page          string/f]
                           [noun-singular        string/f]
                           [note-count           integer/f]
                           [doc-html             string/f]
                           [content-html         string/f]
                           [disposition          string/f]
                           [disp-html-anchor     string/f]
                           [listing-full-html    string/f]
                           [listing-excerpt-html string/f]
                           [listing-short-html   string/f])
            #:constructor-name make-cache:article]{

226
227
228
229
230
231
232
233
234
235






236
237
238
239



240
241
234
235
236
237
238
239
240



241
242
243
244
245
246
247
248


249
250
251
252
253







-
-
-
+
+
+
+
+
+


-
-
+
+
+


                               [html-anchor string/f])
                              #:constructor-name make-cache:index-entry]{
                              
Table holding cached information about index entries found in articles.

}

@defstruct*[listing ([html        string/f]
                     [published   date/f]
                     [series-page symbol/f])
@defstruct*[listing ([path        string/f]
                     [title       string/f]
                     [author      string/f]
                     [published   string/f]
                     [updated     string/f]
                     [html        string/f])
                    #:constructor-name make-listing]{

This is not a table that persists in the cache database; rather it is the schema targeted by
@racket[articles] and @racket[articles+notes] using deta’s @racket[project-onto].
This is a “virtual” schema targeted by @racket[articles] and @racket[articles+notes] using deta’s
@racket[project-onto]. It supplies the minimum set of fields needed to build the RSS feed; most
times (e.g., on @tech{series} pages) only the @tt{html} field is used, via @racket[fenced-listing].

}

Modified crystalize.rkt from [783ca339] to [c9517191].

56
57
58
59
60
61
62
63

64
65
66
67
68
69
70
56
57
58
59
60
61
62

63
64
65
66
67
68
69
70







-
+







                  #:published pubdate
                  #:updated (maybe-meta 'updated)
                  #:author (maybe-meta 'author default-authorname)
                  #:conceal (maybe-meta 'conceal)
                  #:series-page series-node
                  #:noun-singular (maybe-meta 'noun (series-metas-noun))
                  #:note-count (length note-txprs)
                  #:doc-html doc-html
                  #:content-html doc-html
                  #:disposition disposition
                  #:disp-html-anchor disp-note-id
                  #:listing-full-html (string-append header doc-html footer)
                  #:listing-excerpt-html ""
                  #:listing-short-html listing-short))
    (values title-plain (string-append header doc-html notes-section-html footer))))

Modified rss-feed.rkt from [c66da8d8] to [1e1d5b0c].

1
2
3
4
5
6
7
8

9
10
11
12
13

14
15
16
17
18
19
20
1
2
3
4
5
6
7
8
9
10
11
12
13

14
15
16
17
18
19
20
21








+




-
+







#lang racket/base

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

;; Generates an Atom feed from the SQLite cache

(require txexpr
         deta
         racket/match
         racket/file
         racket/date
         racket/string
         db/base
         racket/sequence
         "dust.rkt"
         "cache.rkt")

(provide main)

(define feed-author default-authorname)
(define feed-author-email "joel@jdueck.net")
41
42
43
44
45
46
47
48
49
50
51
52
53

54
55
56
57
58
59
60

61
62
63
64
65
66
67
68
69

70
71
72


73
74
75
76

77
78
79
80
81
82





83
84

85
86

87
88
89
90
91
92
93
94
95
96
97
98
99

100
101
102
103
104
42
43
44
45
46
47
48






49







50









51
52


53
54

55


56






57
58
59
60
61
62

63
64

65
66
67
68
69
70
71
72
73
74
75
76
77

78
79
80
81
82
83







-
-
-
-
-
-
+
-
-
-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
+

-
-
+
+
-

-
-
+
-
-
-
-
-
-
+
+
+
+
+

-
+

-
+












-
+





  (define timestamp
    (parameterize [(date-display-format 'iso-8601)]
      (date->string now #t)))
  (string-append timestamp "Z"))

;; Get the data out of the SQLite cache as vectors
(define (fetch-rows)
  (define fields '(pagenode title_plain published updated author doc_html))
  (define select #<<---
     SELECT `path`, `title`, `published`, `updated`, `author`, `entry_contents` FROM
       (SELECT `page` AS `path`,
               `title_plain` AS `title`,
               `published`,
  (sequence->list
               `updated`,
               `author`,
               `doc_html` AS `entry_contents`
        FROM `articles` WHERE (NOT (`conceal` LIKE "%all%")) AND (NOT (`conceal` LIKE "%feed%"))
        UNION
        SELECT `page` || '#' || `html_anchor` AS `path`,
               `title_plain` AS `title`,
   (in-entities (cache-conn)
               `published`,
               "" AS `updated`,
               `author`,
               `content_html` as `entry_contents`
        FROM `notes` WHERE (NOT (`conceal` LIKE "%all%")) AND (NOT (`conceal` LIKE "%feed%")))
        ORDER BY `published` DESC LIMIT ~a
---
    )
  (query-rows (cache-conn) (format select feed-item-limit)))
                (articles+notes 'content #:series #f #:limit feed-item-limit))))

(define (vector->rss-item vec)
  (match-define
(define (listing->rss-item lst)
  (match-define (listing _ path title author published updated html) lst)
    (vector path title published updated author contents) vec)
  (define entry-url (string-append feed-site-url web-root path))
  (define update-ts
    (cond [(non-empty-string? updated) updated]
  (define updated-ts (if (non-empty-string? updated) updated published))
          [else published]))
    
  `(entry (author    (name ,author))
          (published ,(ymd->rfc3339 published))
          (updated   ,(ymd->rfc3339 update-ts))
          (title     ,title)
  
  `(entry (author (name ,author))
          (published    ,(ymd->rfc3339 published))
          (updated      ,(ymd->rfc3339 updated-ts))
          (title        ,title)
          (link [[rel "alternate"] [href ,entry-url]])
          (id        ,entry-url)
          (id           ,entry-url)
          (summary [[type "html"]]
                   ,(as-cdata contents))))
                   ,(as-cdata html))))

(define (rss-feed)
  (define feed-xpr
    `(feed [[xml:lang "en-us"] [xmlns "http://www.w3.org/2005/Atom"]]
           (title ,feed-title)
           (link [[rel "self"] [href ,(string-append feed-site-url web-root "feed.xml")]])
           (generator [[uri "http://pollenpub.com/"]] "Pollen")
           (id ,(string-append feed-site-url web-root))
           (updated ,(current-rfc3339))
           (author
            (name ,feed-author)
            (email ,@(email-encode feed-author-email)))
           ,@(map vector->rss-item (fetch-rows))))
           ,@(map listing->rss-item (fetch-rows))))
  (string-append "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
                 (xexpr->string feed-xpr)))

(define (main)
  (display-to-file (rss-feed) "feed.xml" #:mode 'text #:exists 'replace))