From 7bc1c41a598100fa451919de65600f9af369a9f0 Mon Sep 17 00:00:00 2001 From: Andros Fenollosa Date: Thu, 28 Oct 2021 09:58:34 +0200 Subject: [PATCH] Add function --- project.clj | 2 +- resources/themes/clojure/index.html | 2 +- resources/themes/dark/index.html | 2 +- resources/themes/light/index.html | 2 +- resources/themes/sepia/index.html | 2 +- src/rsspaper/feeds.clj | 56 +++++++++++++++++------------ 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/project.clj b/project.clj index c818f2c..1b86d39 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject rsspaper "1.2.0" +(defproject rsspaper "1.2.1" :description "RSSpaper" :url "https://github.com/tanrax/RSSpaper" :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" diff --git a/resources/themes/clojure/index.html b/resources/themes/clojure/index.html index b6bb381..3b49936 100644 --- a/resources/themes/clojure/index.html +++ b/resources/themes/clojure/index.html @@ -7,7 +7,7 @@ - + diff --git a/resources/themes/dark/index.html b/resources/themes/dark/index.html index 918ffe6..6011399 100644 --- a/resources/themes/dark/index.html +++ b/resources/themes/dark/index.html @@ -7,7 +7,7 @@ - + diff --git a/resources/themes/light/index.html b/resources/themes/light/index.html index 08c227b..661ec33 100644 --- a/resources/themes/light/index.html +++ b/resources/themes/light/index.html @@ -7,7 +7,7 @@ - + diff --git a/resources/themes/sepia/index.html b/resources/themes/sepia/index.html index 562af5f..f316461 100644 --- a/resources/themes/sepia/index.html +++ b/resources/themes/sepia/index.html @@ -7,7 +7,7 @@ - + diff --git a/src/rsspaper/feeds.clj b/src/rsspaper/feeds.clj index 9cd34cf..5aebc55 100644 --- a/src/rsspaper/feeds.clj +++ b/src/rsspaper/feeds.clj @@ -35,20 +35,32 @@ ;; Add in every article, all information from feed (concat articles (map (fn [article] (assoc article :feed (:feed (update-in feed [:feed] dissoc :entries)))) (get-in feed [:feed :entries])))) [] feeds)) + +(defn add-domain-to-relative-path + [url-complete url-relative] + ;; Converts a relative path to a path with its domain. + ;; /foo/boo/ -> example.com/foo/boo/ + (let [is-relative (= (str (first url-relative)) "/") + url-elements (re-find #"(.+\/\/|www.)(.*?)\/.+" url-complete) + url-with-domain (if is-relative (str (get url-elements 1) (get url-elements 2) url-relative) url-relative)] + url-with-domain)) + + (defn add-cover-article [articles] ;; Add cover to article search first image in description ;; Iterate every blog (map (fn [article] ; User feedback - (prn (str "Looking for cover image for article > " (:link article))) + (prn (str "Looking for cover image for article > " (:feed-url article))) ; Search cover image - (let [url-article (:link article) + (let [url-article (add-domain-to-relative-path (:feed-url article) (:link article)) html (:body (client/get url-article {:insecure? true :throw-exceptions false})) url-og-image (second (re-find #"].*?property=\"og:image(?::url)?\".*?content=\"(.*?)\".*?>|].*?content=\"(.*?)\".*?property=\"og:image(?::url)?\".*?>" html)) url-first-image (second (re-find #"[\s\S]+]+src=\"([^\">]+)\"|id=['\"] ?main ?['\"]>[\s\S]+]+src=\"([^\">]+)\"|class=['\"] ?main ?[\'\"]>[\s\S]+]+src=\"([^\">]+)\"" html)) images [url-og-image url-first-image] - url-final-image (first (filter (fn [item] (not (nil? item))) images))] + url-valid (first (filter (fn [item] (not (nil? item))) images)) + url-final-image (add-domain-to-relative-path url-article url-valid)] (assoc article :cover url-final-image))) articles)) (defn order-published @@ -60,22 +72,22 @@ [] ;; Get all feeds from config -> feeds (-> - (reduce - (fn [feeds feed-url] - ; Read feed - (let [feed (parse-url feed-url {:insecure? true :throw-exceptions false})] - ; User feedback - (prn (str "Reading RSS > " feed-url)) - ; Check is not null - (if-not (nil? feed) - ; Add feed - (conj feeds feed) - ; Alert fail - (prn (str "Error with '" feed-url) "'")))) - [] (:feeds config)) - zip-feeds-in-articles - datetimes-to-unixtime - filter-edition - order-published - add-cover-article - add-datetimes-formatter)) + (reduce + (fn [feeds feed-url] + ; Read feed + (let [feed (parse-url feed-url {:insecure? true :throw-exceptions false})] + ; User feedback + (prn (str "Reading RSS > " feed-url)) + ; Check is not null + (if-not (nil? feed) + ; Add feed and add key feed original + (conj feeds (assoc feed :feed-url feed-url)) + ; Alert fail + (prn (str "Error with '" feed-url) "'")))) + [] (:feeds config)) + zip-feeds-in-articles + datetimes-to-unixtime + filter-edition + order-published + add-cover-article + add-datetimes-formatter))