From ac8fb30fc7af20e564daf8389d8d3b66fc7a9093 Mon Sep 17 00:00:00 2001 From: tfa Date: Fri, 25 Mar 2022 15:19:01 +0100 Subject: [PATCH] Ajout fichiers --- .gitignore | 4 + README.md | 6 ++ config_example.yaml | 5 + rss2wallabag.go | 217 ++++++++++++++++++++++++++++++++++++++++++++ sites.yaml | 12 +++ 5 files changed, 244 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 config_example.yaml create mode 100644 rss2wallabag.go create mode 100755 sites.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc72aa4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +config.yaml +info.log +*.sh +rss2wallabag diff --git a/README.md b/README.md new file mode 100644 index 0000000..38a32ab --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# Rss2Wallabag + +Script permettant d'aller chercher les nouveaux articles publiés sur un flux RSS et de les envoyer vers Wallabag + +- Patch pour vérifier que les articles Mediaparts pour éviter les doublons +- Changement du titre des articles Mediaparts pour ne pas utiliser le titre raccourci du flux diff --git a/config_example.yaml b/config_example.yaml new file mode 100755 index 0000000..18c9abb --- /dev/null +++ b/config_example.yaml @@ -0,0 +1,5 @@ +host: https://wallabag.host.com +client_id: id +client_secret: secret +username: user +password: averysecretpassword diff --git a/rss2wallabag.go b/rss2wallabag.go new file mode 100644 index 0000000..3a471b5 --- /dev/null +++ b/rss2wallabag.go @@ -0,0 +1,217 @@ +package main + + import ( + "bytes" + "encoding/json" + "log" + "net/http" + "gopkg.in/yaml.v2" + "io/ioutil" + "github.com/ungerik/go-rss" + "os" + "sync" + "strings" + "time" + "fmt" + ) + + + //Structure de configuration du client Wallabag + type walla struct { + Url string `yaml:"host"` + Client_id string `yaml:"client_id"` + Client_secret string `yaml:"client_secret"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Token string + } + + //Structure rss + type sites struct { + Tags []string `yaml:"tags"` + Url string `yaml:"url"` + } + + type Sites map[string]sites + + //Get configuration from yaml file config.yaml + func (w *walla) getWallaConf() *walla { + + yamlFile, err := ioutil.ReadFile("config.yaml") + if err != nil { + log.Printf("yamlFile.Get err #%v ", err) + } + err = yaml.UnmarshalStrict(yamlFile, w) + if err != nil { + log.Fatalf("Unmarshal: %v", err) + } + return w + } + + //Get rss url and tags + func (c *Sites) getConf() *Sites { + + yamlFile, err := ioutil.ReadFile("sites.yaml") + if err != nil { + log.Printf("yamlFile.Get err #%v ", err) + } + err = yaml.Unmarshal(yamlFile, c) + if err != nil { + log.Fatalf("Unmarshal: %v", err) + } + + return c + } + + //Get wallaba auth token + func (Walla *walla) Auth() *walla { + + message := map[string]interface{}{ + "grant_type":"password", + "client_id":Walla.Client_id, + "client_secret":Walla.Client_secret, + "username":Walla.Username, + "password":Walla.Password, + } + + requestBody,err := json.Marshal(message) + if err != nil { + log.Fatalln(err) + } + resp, err := http.Post(Walla.Url+"/oauth/v2/token", "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + log.Fatalln(err) + } + var result map[string]interface{} + + json.NewDecoder(resp.Body).Decode(&result) + Walla.Token=result["access_token"].(string) + return Walla + } + + //Add a new article + func WallaAddUrl(Walla walla, url string, tags []string, title string) { + + bearer := "Bearer " + Walla.Token + message := map[string]interface{}{ + "url":url, + "tags":strings.Join(tags, ","), + } + requestBody,err := json.Marshal(message) + if err != nil { + log.Fatalln(err) + } + + req, err := http.NewRequest("POST", Walla.Url+"/api/entries.json", bytes.NewBuffer(requestBody)) + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Authorization", bearer) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Fatalln(err) + } + //Correction du titre + var result map[string]interface{} + json.NewDecoder(resp.Body).Decode(&result) + var id float64 + id=result["id"].(float64) + defer resp.Body.Close() + + message = map[string]interface{}{ + "title":title, + } + requestBody,err = json.Marshal(message) + if err != nil { + log.Fatalln(err) + } + + req, err = http.NewRequest("PATCH", Walla.Url+"/api/entries/"+fmt.Sprint(id)+".json", bytes.NewBuffer(requestBody)) + + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Authorization", bearer) + + client = &http.Client{} + resp, err = client.Do(req) + if err != nil { + log.Fatalln(err) + } + defer resp.Body.Close() + +} + + //Check if an article already exists in wallabag + func WallaCheckArticleExists(Walla walla, url string) bool { + + bearer := "Bearer " + Walla.Token + + client := &http.Client{} + + req, err := http.NewRequest("GET", Walla.Url+"/api/entries/exists.json?url="+url, nil) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", bearer) + + + resp, err := client.Do(req) + if err != nil { + log.Fatalln(err) + } + var result map[string]interface{} + + json.NewDecoder(resp.Body).Decode(&result) + defer resp.Body.Close() + return result["exists"].(bool) + } + + //Proccess rss and add new articles + func ProcessRss(k sites, wg *sync.WaitGroup, Walla walla) { + defer wg.Done() + var exists bool + client := &http.Client{Timeout: 25 * time.Second} + + resp, err := rss.ReadWithClient(k.Url,client, false) + if err != nil { + log.Println(err) + } + + channel, err := rss.Regular(resp) + for _, item := range channel.Item { + exists = WallaCheckArticleExists(Walla, item.Link) + //Mediapart fix + if(!exists){ + exists = WallaCheckArticleExists(Walla, item.Link+"?onglet=full") + } + if(exists) { + log.Println(channel.Title+ " : " + item.Title + " Already exists skipping") + } else { + log.Println(channel.Title + " : " + "Adding " + item.Title) + WallaAddUrl(Walla, item.Link, k.Tags, item.Title) + } + } + } + + func main() { + + file, err := os.OpenFile("info.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + log.Fatal(err) + } + defer file.Close() + log.SetOutput(file) + + var MyWalla walla + MyWalla.getWallaConf() + MyWalla.Auth() + + var c Sites + c.getConf() + log.Println(c) + var wg sync.WaitGroup + for k := range c { + wg.Add(1) + go ProcessRss(c[k], &wg, MyWalla) + } + + wg.Wait() + log.Println("All done") + } diff --git a/sites.yaml b/sites.yaml new file mode 100755 index 0000000..4cd9d30 --- /dev/null +++ b/sites.yaml @@ -0,0 +1,12 @@ +mediapart.fr: + tags: + - mediapart + url: https://www.mediapart.fr/articles/feed +ploum.net: + tags: + - ploum + url: http://ploum.net/feed/ +unodieuxconnard.com: + tags: + - od + url: https://unodieuxconnard.com/feed/