Ajout fichiers

.gitignore

@@ -0,0 +1,4 @@
+config.yaml
+info.log
+*.sh
+rss2wallabag

README.md

@@ -0,0 +1,6 @@
+# Rss2Wallabag
+
+Script permettant d'aller chercher les nouveaux articles publiés sur un flux RSS et de les envoyer vers Wallabag
+
+- Patch pour vérifier que les articles Mediaparts pour éviter les doublons
+- Changement du titre des articles Mediaparts pour ne pas utiliser le titre raccourci du flux

config_example.yaml

@@ -0,0 +1,5 @@
+host: https://wallabag.host.com
+client_id: id 
+client_secret: secret 
+username: user
+password: averysecretpassword

rss2wallabag.go

@@ -0,0 +1,217 @@
+package main
+ 
+ import (
+     "bytes"
+     "encoding/json"
+     "log"
+     "net/http"
+     "gopkg.in/yaml.v2"
+     "io/ioutil"
+     "github.com/ungerik/go-rss"
+     "os"
+     "sync"
+     "strings"
+     "time"
+     "fmt"
+ )
+ 
+ 
+ //Structure de configuration du client Wallabag
+ type walla struct {
+     Url           string `yaml:"host"`
+     Client_id     string `yaml:"client_id"`
+     Client_secret string `yaml:"client_secret"`
+     Username     string `yaml:"username"`
+     Password     string `yaml:"password"`
+     Token       string
+ }
+ 
+ //Structure rss
+ type sites struct {
+     Tags []string `yaml:"tags"`
+     Url string `yaml:"url"`
+ }
+ 
+ type Sites map[string]sites
+ 
+ //Get configuration from yaml file config.yaml
+ func (w *walla) getWallaConf() *walla {
+    
+     yamlFile, err := ioutil.ReadFile("config.yaml")
+     if err != nil {
+         log.Printf("yamlFile.Get err   #%v ", err)
+     }   
+     err = yaml.UnmarshalStrict(yamlFile, w)
+     if err != nil {
+         log.Fatalf("Unmarshal: %v", err)
+     }
+     return w
+ }
+ 
+ //Get rss url and tags
+ func (c *Sites) getConf() *Sites {
+ 
+     yamlFile, err := ioutil.ReadFile("sites.yaml")
+     if err != nil {
+         log.Printf("yamlFile.Get err   #%v ", err)
+     }
+     err = yaml.Unmarshal(yamlFile, c)
+     if err != nil {
+         log.Fatalf("Unmarshal: %v", err)
+     }
+ 
+     return c
+ }
+ 
+ //Get wallaba auth token
+ func (Walla *walla) Auth() *walla {
+     
+     message := map[string]interface{}{
+         "grant_type":"password",
+         "client_id":Walla.Client_id,
+         "client_secret":Walla.Client_secret,
+         "username":Walla.Username,
+         "password":Walla.Password,
+     }
+ 
+     requestBody,err := json.Marshal(message)
+     if err != nil {
+         log.Fatalln(err)
+     }
+     resp, err := http.Post(Walla.Url+"/oauth/v2/token", "application/json", bytes.NewBuffer(requestBody))
+     if err != nil {
+         log.Fatalln(err)
+     }
+     var result map[string]interface{}
+ 
+     json.NewDecoder(resp.Body).Decode(&result)
+     Walla.Token=result["access_token"].(string)
+     return Walla
+ }
+ 
+ //Add a new article
+ func WallaAddUrl(Walla walla, url string, tags []string, title string) {
+     
+     bearer := "Bearer " + Walla.Token
+     message := map[string]interface{}{
+         "url":url,
+         "tags":strings.Join(tags, ","),
+     }
+     requestBody,err := json.Marshal(message)
+     if err != nil {
+          log.Fatalln(err)
+     } 
+ 
+     req, err := http.NewRequest("POST", Walla.Url+"/api/entries.json", bytes.NewBuffer(requestBody))
+     req.Header.Set("Content-Type", "application/json")
+     req.Header.Add("Authorization", bearer)
+ 
+     client := &http.Client{}
+     resp, err := client.Do(req)
+     if err != nil {
+         log.Fatalln(err)
+     }
+     //Correction du titre
+     var result map[string]interface{}
+     json.NewDecoder(resp.Body).Decode(&result)
+     var id float64
+     id=result["id"].(float64)
+     defer resp.Body.Close()
+
+     message = map[string]interface{}{
+        "title":title,
+     }
+     requestBody,err = json.Marshal(message)
+     if err != nil {
+          log.Fatalln(err)
+     } 
+ 
+    req, err = http.NewRequest("PATCH", Walla.Url+"/api/entries/"+fmt.Sprint(id)+".json", bytes.NewBuffer(requestBody))
+
+     req.Header.Set("Content-Type", "application/json")
+     req.Header.Add("Authorization", bearer)
+ 
+     client = &http.Client{}
+     resp, err = client.Do(req)
+     if err != nil {
+         log.Fatalln(err)
+     }
+     defer resp.Body.Close()
+ 
+}
+ 
+ //Check if an article already exists in wallabag
+ func WallaCheckArticleExists(Walla walla, url string) bool {
+     
+     bearer := "Bearer " + Walla.Token
+ 
+     client := &http.Client{}
+ 
+     req, err := http.NewRequest("GET", Walla.Url+"/api/entries/exists.json?url="+url, nil)
+     req.Header.Add("Content-Type", "application/json")
+     req.Header.Add("Authorization", bearer)
+ 
+ 
+     resp, err := client.Do(req)   
+     if err != nil {
+         log.Fatalln(err)
+     }
+     var result map[string]interface{}
+ 
+     json.NewDecoder(resp.Body).Decode(&result)
+     defer resp.Body.Close()
+     return result["exists"].(bool)
+ }
+ 
+ //Proccess rss and add new articles
+ func ProcessRss(k sites, wg *sync.WaitGroup, Walla walla) {
+     defer wg.Done()
+     var exists bool
+     client := &http.Client{Timeout: 25 * time.Second}
+ 
+     resp, err := rss.ReadWithClient(k.Url,client, false)
+     if err != nil {
+         log.Println(err)
+     }
+
+     channel, err := rss.Regular(resp)
+     for _, item := range channel.Item {
+        exists = WallaCheckArticleExists(Walla, item.Link)
+        //Mediapart fix
+         if(!exists){
+            exists = WallaCheckArticleExists(Walla, item.Link+"?onglet=full")    
+        }
+        if(exists) {
+            log.Println(channel.Title+ " : " + item.Title + " Already exists skipping")
+        } else {
+            log.Println(channel.Title + " : " + "Adding " + item.Title)
+            WallaAddUrl(Walla, item.Link, k.Tags, item.Title)
+        }
+     }
+ }
+ 
+ func main() {
+ 
+     file, err := os.OpenFile("info.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+     if err != nil {
+         log.Fatal(err)
+     }
+     defer file.Close()
+     log.SetOutput(file)
+ 
+     var MyWalla walla
+     MyWalla.getWallaConf()
+     MyWalla.Auth() 
+    
+     var c Sites
+     c.getConf()
+     log.Println(c)
+     var wg sync.WaitGroup
+     for k := range c {
+         wg.Add(1)
+        go  ProcessRss(c[k], &wg, MyWalla)
+     }
+ 
+     wg.Wait()
+     log.Println("All done")
+ }

sites.yaml

@@ -0,0 +1,12 @@
+mediapart.fr:
+  tags:
+  - mediapart
+  url: https://www.mediapart.fr/articles/feed
+ploum.net:
+  tags:
+  - ploum
+  url: http://ploum.net/feed/
+unodieuxconnard.com:
+  tags:
+  - od
+  url: https://unodieuxconnard.com/feed/