MyFavMirrors
/
RSSPAPER
mirror of https://github.com/tanrax/RSSPAPER.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove futures

This commit is contained in:
Andros Fenollosa 2021-12-16 19:35:14 +01:00
parent ee11925e13
commit 3c63d1b5e0
4 changed files with 30 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -5,4 +5,8 @@ target/
config.yaml config.yaml
log/logger1.log* log/logger1.log*
*.jar *.jar
resources/*.tar resources/*.tar
resources/themes/clojure/static.tar
resources/themes/dark/static.tar
resources/themes/light/static.tar
resources/themes/sepia/static.tar

1
Makefile
View File

@ -3,7 +3,6 @@
build: build:
make build.templates make build.templates
lein uberjar lein uberjar
make rm.statics
echo "Finish!" echo "Finish!"
build.templates: build.templates:

2
project.clj
View File

@ -1,4 +1,4 @@
(defproject rsspaper "1.2.1" (defproject rsspaper "1.2.2"
:description "RSSpaper" :description "RSSpaper"
:url "https://github.com/tanrax/RSSpaper" :url "https://github.com/tanrax/RSSpaper"
:license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"

45
src/rsspaper/feeds.clj
View File

@ -16,13 +16,17 @@
(defn filter-edition (defn filter-edition
[articles] [articles]
(let [daily (c/to-long (t/minus (t/now) (t/days 1))) (let [daily (c/to-long (t/minus (t/now) (t/days 1)))
weekly (c/to-long (t/minus (t/now) (t/weeks 1)))] weekly (c/to-long (t/minus (t/now) (t/weeks 1)))]
(case (:edition config) (case (:edition config)
"daily" (filter (fn [article] (and (not (nil? (:published-date article))) (>= (:published-date article) daily))) articles) "daily" (filter (fn [article] (and (not (nil? (:published-date article))) (>= (:published-date article) daily))) articles)
"weekly" (filter (fn [article] (and (not (nil? (:published-date article))) (>= (:published-date article) weekly))) articles) "weekly" (filter (fn [article] (and (not (nil? (:published-date article))) (>= (:published-date article) weekly))) articles)
articles))) articles)))
(defn remove-future-editions
[articles]
(filter (fn [article] (and (not (nil? (:published-date article))) (< (:published-date article) (c/to-long (t/now))))) articles))
(defn add-datetimes-formatter (defn add-datetimes-formatter
[articles] [articles]
(map (fn [article] (map (fn [article]
@ -35,10 +39,9 @@
;; Add in every article, all information from feed ;; Add in every article, all information from feed
(concat articles (map (fn [article] (assoc (concat articles (map (fn [article] (assoc
;; Add feed-url ;; Add feed-url
(assoc article :feed (assoc article :feed
;; Add feed ;; Add feed
(:feed (update-in feed [:feed] dissoc :entries))) :feed-url (:feed-url feed))) (get-in feed [:feed :entries])))) [] feeds)) (:feed (update-in feed [:feed] dissoc :entries))) :feed-url (:feed-url feed))) (get-in feed [:feed :entries])))) [] feeds))
(defn add-domain-to-relative-path (defn add-domain-to-relative-path
[url-complete url-relative] [url-complete url-relative]
@ -49,7 +52,6 @@
url-with-domain (if is-relative (str (get url-elements 1) (get url-elements 2) url-relative) url-relative)] url-with-domain (if is-relative (str (get url-elements 1) (get url-elements 2) url-relative) url-relative)]
url-with-domain)) url-with-domain))
(defn add-cover-article (defn add-cover-article
[articles] [articles]
;; Add cover to article search first image in description ;; Add cover to article search first image in description
@ -63,7 +65,7 @@
url-og-image (second (re-find #"<meta[^>].*?property=\"og:image(?::url)?\".*?content=\"(.*?)\".*?>|<meta[^>].*?content=\"(.*?)\".*?property=\"og:image(?::url)?\".*?>" html)) url-og-image (second (re-find #"<meta[^>].*?property=\"og:image(?::url)?\".*?content=\"(.*?)\".*?>|<meta[^>].*?content=\"(.*?)\".*?property=\"og:image(?::url)?\".*?>" html))
url-first-image (second (re-find #"<main.*>[\s\S]+<img[^>]+src=\"([^\">]+)\"|id=['\"] ?main ?['\"]>[\s\S]+<img[^>]+src=\"([^\">]+)\"|class=['\"] ?main ?[\'\"]>[\s\S]+<img[^>]+src=\"([^\">]+)\"" html)) url-first-image (second (re-find #"<main.*>[\s\S]+<img[^>]+src=\"([^\">]+)\"|id=['\"] ?main ?['\"]>[\s\S]+<img[^>]+src=\"([^\">]+)\"|class=['\"] ?main ?[\'\"]>[\s\S]+<img[^>]+src=\"([^\">]+)\"" html))
images [url-og-image url-first-image] images [url-og-image url-first-image]
url-valid (first (filter (fn [item] (not (nil? item))) images)) url-valid (first (remove nil? images))
url-final-image (add-domain-to-relative-path (:feed-url article) url-valid)] url-final-image (add-domain-to-relative-path (:feed-url article) url-valid)]
(assoc article :cover url-final-image))) articles)) (assoc article :cover url-final-image))) articles))
@ -76,22 +78,23 @@
[] []
;; Get all feeds from config -> feeds ;; Get all feeds from config -> feeds
(-> (->
(reduce (reduce
(fn [feeds feed-url] (fn [feeds feed-url]
; Read feed ; Read feed
(let [feed (parse-url feed-url {:insecure? true :throw-exceptions false})] (let [feed (parse-url feed-url {:insecure? true :throw-exceptions false})]
; User feedback ; User feedback
(prn (str "Reading RSS > " feed-url)) (prn (str "Reading RSS > " feed-url))
; Check is not null ; Check is not null
(if-not (nil? feed) (if-not (nil? feed)
; Add feed and add key feed original ; Add feed and add key feed original
(conj feeds (assoc feed :feed-url feed-url)) (conj feeds (assoc feed :feed-url feed-url))
; Alert fail ; Alert fail
(prn (str "Error with '" feed-url) "'")))) (prn (str "Error with '" feed-url) "'"))))
[] (:feeds config)) [] (:feeds config))
zip-feeds-in-articles zip-feeds-in-articles
datetimes-to-unixtime datetimes-to-unixtime
filter-edition filter-edition
order-published remove-future-editions
add-cover-article order-published
add-datetimes-formatter)) add-cover-article
add-datetimes-formatter))