◊(Local Yarn Code "Check-in [d3755213]")

Overview
Comment:Build up listing schema, use for RSS feed ([2f082139])
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: d37552133761dc80fd424f07d8d82c03f335eb6f52933a51c3de14842ccb1dfa
User & Date: joel on 2020-02-21 22:43:34
Other Links: manifest | tags
Context
2020-02-22
00:04
Minor code doc edits check-in: e538e325 user: joel tags: trunk
2020-02-21
22:43
Build up listing schema, use for RSS feed ([2f082139]) check-in: d3755213 user: joel tags: trunk
2020-02-20
08:30
Update code docs to reflect [47d9eea54] check-in: b223c8a5 user: joel tags: trunk
Changes

Modified cache.rkt from [fa204a09] to [eeff2fe2].

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
..
86
87
88
89
90
91
92
93


94
95


96
97
98
99
100
101
102
...
137
138
139
140
141
142
143



144
145
146



147
148
149


150
151
152
153
154
155
156
157



158
159
160
161




162


163
164
165
166
167




168


169
170
171
172
173
174
175
176
177
   [published            string/f]
   [updated              string/f]
   [author               string/f]
   [conceal              string/f]
   [series-page          symbol/f]
   [noun-singular        string/f]
   [note-count           integer/f]
   [doc-html             string/f]
   [disposition          string/f]
   [disp-html-anchor     string/f]
   [listing-full-html    string/f]   ; full content but without notes
   [listing-excerpt-html string/f]   ; Not used for now
   [listing-short-html   string/f])) ; Date and title only

(define-schema cache:note #:table "notes"
................................................................................
   [entry       string/f]
   [subentry    string/f]
   [page        symbol/f]
   [html-anchor string/f]))

(define-schema listing
  #:virtual
  ([html        string/f]


   [published   date/f]
   [series-page symbol/f]))



(define (init-cache-db!)
  (create-table! (cache-conn) 'cache:article)
  (create-table! (cache-conn) 'cache:note)
  (create-table! (cache-conn) 'cache:series)
  (create-table! (cache-conn) 'cache:index-entry))

................................................................................

;; Needed to "parameterize" column names
;; see https://github.com/Bogdanp/deta/issues/14#issuecomment-573344928
(require (prefix-in ast: deta/private/ast))

;; Builds a query to fetch articles
(define (articles type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])



  (define html-field (format "listing_~a_html" type))
  (~> (from cache:article #:as a)
      (select (fragment (ast:as (ast:qualified "a" html-field) "html"))



              a.published
              a.series-page
              a.conceal)


      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

;; Builds a query that returns articles and notes intermingled chronologically
(define (articles+notes type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])



  (define html-field (format "listing_~a_html" type))
  (~> (from (subquery
             (~> (from cache:article #:as A)
                 (select (fragment (ast:as (ast:qualified "A" html-field) "html"))




                         A.published


                         A.series-page
                         A.conceal)
                 (union
                  (~> (from cache:note #:as N)
                      (select (fragment (ast:as (ast:qualified "N" html-field) "html"))




                              N.published


                              N.series-page
                              N.conceal)))))
            #:as a)
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))








|







 







|
>
>
|
<
>
>







 







>
>
>
|

<
>
>
>

<
<
>
>








>
>
>
|


<
>
>
>
>
|
>
>
|
|


<
>
>
>
>
|
>
>
|
|







45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
..
86
87
88
89
90
91
92
93
94
95
96

97
98
99
100
101
102
103
104
105
...
140
141
142
143
144
145
146
147
148
149
150
151

152
153
154
155


156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171

172
173
174
175
176
177
178
179
180
181
182

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
   [published            string/f]
   [updated              string/f]
   [author               string/f]
   [conceal              string/f]
   [series-page          symbol/f]
   [noun-singular        string/f]
   [note-count           integer/f]
   [content-html             string/f]
   [disposition          string/f]
   [disp-html-anchor     string/f]
   [listing-full-html    string/f]   ; full content but without notes
   [listing-excerpt-html string/f]   ; Not used for now
   [listing-short-html   string/f])) ; Date and title only

(define-schema cache:note #:table "notes"
................................................................................
   [entry       string/f]
   [subentry    string/f]
   [page        symbol/f]
   [html-anchor string/f]))

(define-schema listing
  #:virtual
  ([path        string/f]
   [title       string/f]
   [author      string/f]
   [published   string/f]

   [updated     string/f]
   [html        string/f]))

(define (init-cache-db!)
  (create-table! (cache-conn) 'cache:article)
  (create-table! (cache-conn) 'cache:note)
  (create-table! (cache-conn) 'cache:series)
  (create-table! (cache-conn) 'cache:index-entry))

................................................................................

;; Needed to "parameterize" column names
;; see https://github.com/Bogdanp/deta/issues/14#issuecomment-573344928
(require (prefix-in ast: deta/private/ast))

;; Builds a query to fetch articles
(define (articles type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])
  (define html-field
    (match type
      ['content "content_html"]
      [_ (format "listing_~a_html" type)]))
  (~> (from cache:article #:as a)

      (select (as a.page path)
              (as a.title-plain title)
              a.author
              a.published


              a.updated
              (fragment (ast:as (ast:qualified "a" html-field) "html")))
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

;; Builds a query that returns articles and notes intermingled chronologically
(define (articles+notes type #:series [s #t] #:limit [lim -1] #:order [ord 'desc])
  (define html-field
    (match type
      ['content "content_html"]
      [_ (format "listing_~a_html" type)]))
  (~> (from (subquery
             (~> (from cache:article #:as A)

                 (select 
                  (as A.page path)
                  (as A.title-plain title)
                  A.author
                  A.published
                  A.updated
                  (fragment (ast:as (ast:qualified "A" html-field) "html"))
                  A.series-page
                  A.conceal)
                 (union
                  (~> (from cache:note #:as N)

                      (select 
                       (as (array-concat N.page "#" N.html-anchor) path)
                       (as N.title-plain title)
                       N.author
                       N.published
                       (as "" updated) 
                       (fragment (ast:as (ast:qualified "N" html-field) "html"))
                       N.series-page
                       N.conceal)))))
            #:as a)
      (where-series s)
      (where-not-concealed)
      (limit ,lim)
      (order-by ([a.published ,ord]))
      (project-onto listing-schema)))

Modified code-docs/cache.scrbl from [bc3bc40e] to [1876eb41].

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108







109
110
111
112
113
114
115
116
117
118
119
120
121
122

123
124
125
126
127
128
129
...
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
...
226
227
228
229
230
231
232
233


234
235


236
237
238
239


240
241
Returns the HTML bodies for the articles and/or notes returned by @racket[_listing-query] as a list
of strings. The @racket[_query] will usually be the result of a call to @racket[articles] or
@racket[articles+notes], but can be any custom query that projects onto the @racket[listing] schema
(see @racket[project-onto]).

}

@deftogether[(@defproc[(articles [type (or/c 'full 'excerpt 'short)]
                                 [#:series series (or/c string? (listof string?) boolean?) #t]
                                 [#:limit limit integer? -1]
                                 [order stringish? 'desc]) query?]
              @defproc[(articles+notes [type (or/c 'full 'excerpt 'short)]
                                       [#:series series (or/c string? (listof string?) boolean?) #t]
                                       [#:limit limit integer? -1]
                                       [order stringish? 'desc]) query?])]{

Create a query that fetches either articles only, or articles and notes intermingled, respectively.
The results will be sorted by publish date according to @racket[_order] and optionally limited to
a particular series. Use the resulting query with the @racket[listing-htmls] or
@racket[fenced-listing] functions provided by this module, or with deta’s @racket[in-entities] if
you want to work with the @racket[listing] schema structs.








If @racket[_series] expression evaluates to @racket[#f], articles will not be filtered by series. If
it evaluates to @racket[#t] (the default), articles will be filtered by those that specify the
current output of @racket[here-output-path] in their @tt{series_pagenode} column in the SQLite
cache. If a string or a symbol is supplied, articles will be filtered by those containing the result
of @racket[(format "series/~a.html" _series)] in their @tt{series_pagenode} column in the SQLite
cache. If a list of strings or symbols is provided, this @racket[format] operation will be applied
to each of its members and articles whose @tt{series_pagenode} column matches any of the resulting
values will be included.

The @racket[_order] expression must evaluate to either @racket["ASC"] or @racket["DESC"] and the
@racket[_limit] expressions must evaluate to a value suitable for use in the @tt{LIMIT} clause of
@ext-link["https://sqlite.org/lang_select.html"]{a SQLite @tt{SELECT} statement}. An expression that
evaluates to a negative integer (the default) is the same as having no limit.


Typically you will pass these functions by name to listing functions like @racket[fenced-listing]
rather than calling them directly.

@examples[#:eval example-eval
(articles 'full)]
}
................................................................................
                           [published            string/f]
                           [updated              string/f]
                           [author               string/f]
                           [conceal              string/f]
                           [series-page          string/f]
                           [noun-singular        string/f]
                           [note-count           integer/f]
                           [doc-html             string/f]
                           [disposition          string/f]
                           [disp-html-anchor     string/f]
                           [listing-full-html    string/f]
                           [listing-excerpt-html string/f]
                           [listing-short-html   string/f])
            #:constructor-name make-cache:article]{

................................................................................
                               [html-anchor string/f])
                              #:constructor-name make-cache:index-entry]{
                              
Table holding cached information about index entries found in articles.

}

@defstruct*[listing ([html        string/f]


                     [published   date/f]
                     [series-page symbol/f])


                    #:constructor-name make-listing]{

This is not a table that persists in the cache database; rather it is the schema targeted by
@racket[articles] and @racket[articles+notes] using deta’s @racket[project-onto].



}







|



|









>
>
>
>
>
>
>










|
|
|
|
>







 







|







 







|
>
>
|
<
>
>


<
|
>
>


88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
...
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
...
234
235
236
237
238
239
240
241
242
243
244

245
246
247
248

249
250
251
252
253
Returns the HTML bodies for the articles and/or notes returned by @racket[_listing-query] as a list
of strings. The @racket[_query] will usually be the result of a call to @racket[articles] or
@racket[articles+notes], but can be any custom query that projects onto the @racket[listing] schema
(see @racket[project-onto]).

}

@deftogether[(@defproc[(articles [type (or/c 'full 'excerpt 'short 'content)]
                                 [#:series series (or/c string? (listof string?) boolean?) #t]
                                 [#:limit limit integer? -1]
                                 [order stringish? 'desc]) query?]
              @defproc[(articles+notes [type (or/c 'full 'excerpt 'short 'content)]
                                       [#:series series (or/c string? (listof string?) boolean?) #t]
                                       [#:limit limit integer? -1]
                                       [order stringish? 'desc]) query?])]{

Create a query that fetches either articles only, or articles and notes intermingled, respectively.
The results will be sorted by publish date according to @racket[_order] and optionally limited to
a particular series. Use the resulting query with the @racket[listing-htmls] or
@racket[fenced-listing] functions provided by this module, or with deta’s @racket[in-entities] if
you want to work with the @racket[listing] schema structs.

The @racket[_type] parameter specifies what version of the articles’ and notes’ HTML markup you
want. For HTML suitable for listing several articles and/or notes together on the same page, use
@racket['full] (the full content but not including @tech{notes}), @racket['excerpt] (like full but
abbreviated to only the excerpt if one was specified) or @racket['short] (date and title only). Use
@racket['content] to get the entire HTML content, including any notes but not including any header
or footer. (This is the option used in the RSS feed.)

If @racket[_series] expression evaluates to @racket[#f], articles will not be filtered by series. If
it evaluates to @racket[#t] (the default), articles will be filtered by those that specify the
current output of @racket[here-output-path] in their @tt{series_pagenode} column in the SQLite
cache. If a string or a symbol is supplied, articles will be filtered by those containing the result
of @racket[(format "series/~a.html" _series)] in their @tt{series_pagenode} column in the SQLite
cache. If a list of strings or symbols is provided, this @racket[format] operation will be applied
to each of its members and articles whose @tt{series_pagenode} column matches any of the resulting
values will be included.

The @racket[_order] expression must evaluate to either @racket["ASC"] or @racket["DESC"] (or
equivalent symbols) and the @racket[_limit] expressions must evaluate to a value suitable for use in
the @tt{LIMIT} clause of @ext-link["https://sqlite.org/lang_select.html"]{a SQLite @tt{SELECT}
statement}. An expression that evaluates to a negative integer (the default) is the same as having
no limit.

Typically you will pass these functions by name to listing functions like @racket[fenced-listing]
rather than calling them directly.

@examples[#:eval example-eval
(articles 'full)]
}
................................................................................
                           [published            string/f]
                           [updated              string/f]
                           [author               string/f]
                           [conceal              string/f]
                           [series-page          string/f]
                           [noun-singular        string/f]
                           [note-count           integer/f]
                           [content-html         string/f]
                           [disposition          string/f]
                           [disp-html-anchor     string/f]
                           [listing-full-html    string/f]
                           [listing-excerpt-html string/f]
                           [listing-short-html   string/f])
            #:constructor-name make-cache:article]{

................................................................................
                               [html-anchor string/f])
                              #:constructor-name make-cache:index-entry]{
                              
Table holding cached information about index entries found in articles.

}

@defstruct*[listing ([path        string/f]
                     [title       string/f]
                     [author      string/f]
                     [published   string/f]

                     [updated     string/f]
                     [html        string/f])
                    #:constructor-name make-listing]{


This is a “virtual” schema targeted by @racket[articles] and @racket[articles+notes] using deta’s
@racket[project-onto]. It supplies the minimum set of fields needed to build the RSS feed; most
times (e.g., on @tech{series} pages) only the @tt{html} field is used, via @racket[fenced-listing].

}

Modified crystalize.rkt from [783ca339] to [c9517191].

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
                  #:published pubdate
                  #:updated (maybe-meta 'updated)
                  #:author (maybe-meta 'author default-authorname)
                  #:conceal (maybe-meta 'conceal)
                  #:series-page series-node
                  #:noun-singular (maybe-meta 'noun (series-metas-noun))
                  #:note-count (length note-txprs)
                  #:doc-html doc-html
                  #:disposition disposition
                  #:disp-html-anchor disp-note-id
                  #:listing-full-html (string-append header doc-html footer)
                  #:listing-excerpt-html ""
                  #:listing-short-html listing-short))
    (values title-plain (string-append header doc-html notes-section-html footer))))








|







56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
                  #:published pubdate
                  #:updated (maybe-meta 'updated)
                  #:author (maybe-meta 'author default-authorname)
                  #:conceal (maybe-meta 'conceal)
                  #:series-page series-node
                  #:noun-singular (maybe-meta 'noun (series-metas-noun))
                  #:note-count (length note-txprs)
                  #:content-html doc-html
                  #:disposition disposition
                  #:disp-html-anchor disp-note-id
                  #:listing-full-html (string-append header doc-html footer)
                  #:listing-excerpt-html ""
                  #:listing-short-html listing-short))
    (values title-plain (string-append header doc-html notes-section-html footer))))

Modified rss-feed.rkt from [c66da8d8] to [1e1d5b0c].

2
3
4
5
6
7
8

9
10
11
12
13
14
15
16
17
18
19
20
..
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

;; Generates an Atom feed from the SQLite cache

(require txexpr

         racket/match
         racket/file
         racket/date
         racket/string
         db/base
         "dust.rkt"
         "cache.rkt")

(provide main)

(define feed-author default-authorname)
(define feed-author-email "joel@jdueck.net")
................................................................................
  (define timestamp
    (parameterize [(date-display-format 'iso-8601)]
      (date->string now #t)))
  (string-append timestamp "Z"))

;; Get the data out of the SQLite cache as vectors
(define (fetch-rows)
  (define fields '(pagenode title_plain published updated author doc_html))
  (define select #<<---
     SELECT `path`, `title`, `published`, `updated`, `author`, `entry_contents` FROM
       (SELECT `page` AS `path`,
               `title_plain` AS `title`,
               `published`,
               `updated`,
               `author`,
               `doc_html` AS `entry_contents`
        FROM `articles` WHERE (NOT (`conceal` LIKE "%all%")) AND (NOT (`conceal` LIKE "%feed%"))
        UNION
        SELECT `page` || '#' || `html_anchor` AS `path`,
               `title_plain` AS `title`,
               `published`,
               "" AS `updated`,
               `author`,
               `content_html` as `entry_contents`
        FROM `notes` WHERE (NOT (`conceal` LIKE "%all%")) AND (NOT (`conceal` LIKE "%feed%")))
        ORDER BY `published` DESC LIMIT ~a
---
    )
  (query-rows (cache-conn) (format select feed-item-limit)))

(define (vector->rss-item vec)
  (match-define
    (vector path title published updated author contents) vec)

  (define entry-url (string-append feed-site-url web-root path))
  (define update-ts
    (cond [(non-empty-string? updated) updated]
          [else published]))
    
  `(entry (author    (name ,author))
          (published ,(ymd->rfc3339 published))
          (updated   ,(ymd->rfc3339 update-ts))
          (title     ,title)
          (link [[rel "alternate"] [href ,entry-url]])
          (id        ,entry-url)
          (summary [[type "html"]]
                   ,(as-cdata contents))))

(define (rss-feed)
  (define feed-xpr
    `(feed [[xml:lang "en-us"] [xmlns "http://www.w3.org/2005/Atom"]]
           (title ,feed-title)
           (link [[rel "self"] [href ,(string-append feed-site-url web-root "feed.xml")]])
           (generator [[uri "http://pollenpub.com/"]] "Pollen")
           (id ,(string-append feed-site-url web-root))
           (updated ,(current-rfc3339))
           (author
            (name ,feed-author)
            (email ,@(email-encode feed-author-email)))
           ,@(map vector->rss-item (fetch-rows))))
  (string-append "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
                 (xexpr->string feed-xpr)))

(define (main)
  (display-to-file (rss-feed) "feed.xml" #:mode 'text #:exists 'replace))







>




|







 







|
|
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
|
<
<
>

<
|
<
|
|
|
|
|

|

|












|





2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
..
42
43
44
45
46
47
48
49
50
51

















52


53


54
55

56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

; SPDX-License-Identifier: BlueOak-1.0.0
; This file is licensed under the Blue Oak Model License 1.0.0.

;; Generates an Atom feed from the SQLite cache

(require txexpr
         deta
         racket/match
         racket/file
         racket/date
         racket/string
         racket/sequence
         "dust.rkt"
         "cache.rkt")

(provide main)

(define feed-author default-authorname)
(define feed-author-email "joel@jdueck.net")
................................................................................
  (define timestamp
    (parameterize [(date-display-format 'iso-8601)]
      (date->string now #t)))
  (string-append timestamp "Z"))

;; Get the data out of the SQLite cache as vectors
(define (fetch-rows)
  (sequence->list
   (in-entities (cache-conn)
                (articles+notes 'content #:series #f #:limit feed-item-limit))))




















(define (listing->rss-item lst)


  (match-define (listing _ path title author published updated html) lst)
  (define entry-url (string-append feed-site-url web-root path))

  (define updated-ts (if (non-empty-string? updated) updated published))

  
  `(entry (author (name ,author))
          (published    ,(ymd->rfc3339 published))
          (updated      ,(ymd->rfc3339 updated-ts))
          (title        ,title)
          (link [[rel "alternate"] [href ,entry-url]])
          (id           ,entry-url)
          (summary [[type "html"]]
                   ,(as-cdata html))))

(define (rss-feed)
  (define feed-xpr
    `(feed [[xml:lang "en-us"] [xmlns "http://www.w3.org/2005/Atom"]]
           (title ,feed-title)
           (link [[rel "self"] [href ,(string-append feed-site-url web-root "feed.xml")]])
           (generator [[uri "http://pollenpub.com/"]] "Pollen")
           (id ,(string-append feed-site-url web-root))
           (updated ,(current-rfc3339))
           (author
            (name ,feed-author)
            (email ,@(email-encode feed-author-email)))
           ,@(map listing->rss-item (fetch-rows))))
  (string-append "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
                 (xexpr->string feed-xpr)))

(define (main)
  (display-to-file (rss-feed) "feed.xml" #:mode 'text #:exists 'replace))