rss2wallabag/rss2wallabag.go

package main
 
 import (
     "bytes"
     "encoding/json"
     "log"
     "net/http"
     "gopkg.in/yaml.v2"
     "io/ioutil"
     "github.com/ungerik/go-rss"
     "os"
     "sync"
     "strings"
     "time"
     "fmt"
 )
 
 
 //Structure de configuration du client Wallabag
 type walla struct {
     Url           string `yaml:"host"`
     Client_id     string `yaml:"client_id"`
     Client_secret string `yaml:"client_secret"`
     Username     string `yaml:"username"`
     Password     string `yaml:"password"`
     Token       string
 }
 
 //Structure rss
 type sites struct {
     Tags []string `yaml:"tags"`
     Url string `yaml:"url"`
 }
 
 type Sites map[string]sites
 
 //Get configuration from yaml file config.yaml
 func (w *walla) getWallaConf() *walla {
    
     yamlFile, err := ioutil.ReadFile("config.yaml")
     if err != nil {
         log.Printf("yamlFile.Get err   #%v ", err)
     }   
     err = yaml.UnmarshalStrict(yamlFile, w)
     if err != nil {
         log.Fatalf("Unmarshal: %v", err)
     }
     return w
 }
 
 //Get rss url and tags
 func (c *Sites) getConf() *Sites {
 
     yamlFile, err := ioutil.ReadFile("sites.yaml")
     if err != nil {
         log.Printf("yamlFile.Get err   #%v ", err)
     }
     err = yaml.Unmarshal(yamlFile, c)
     if err != nil {
         log.Fatalf("Unmarshal: %v", err)
     }
 
     return c
 }
 
 //Get wallaba auth token
 func (Walla *walla) Auth() *walla {
     
     message := map[string]interface{}{
         "grant_type":"password",
         "client_id":Walla.Client_id,
         "client_secret":Walla.Client_secret,
         "username":Walla.Username,
         "password":Walla.Password,
     }
 
     requestBody,err := json.Marshal(message)
     if err != nil {
         log.Fatalln(err)
     }
     resp, err := http.Post(Walla.Url+"/oauth/v2/token", "application/json", bytes.NewBuffer(requestBody))
     if err != nil {
         log.Fatalln(err)
     }
     var result map[string]interface{}
 
     json.NewDecoder(resp.Body).Decode(&result)
     Walla.Token=result["access_token"].(string)
     return Walla
 }
 
 //Add a new article
 func WallaAddUrl(Walla walla, url string, tags []string, title string) {
     
     bearer := "Bearer " + Walla.Token
     message := map[string]interface{}{
         "url":url,
         "tags":strings.Join(tags, ","),
     }
     requestBody,err := json.Marshal(message)
     if err != nil {
          log.Fatalln(err)
     } 
 
     req, err := http.NewRequest("POST", Walla.Url+"/api/entries.json", bytes.NewBuffer(requestBody))
     req.Header.Set("Content-Type", "application/json")
     req.Header.Add("Authorization", bearer)
 
     client := &http.Client{}
     resp, err := client.Do(req)
     if err != nil {
         log.Fatalln(err)
     }
     //Correction du titre
     var result map[string]interface{}
     json.NewDecoder(resp.Body).Decode(&result)
     var id float64
     id=result["id"].(float64)
     defer resp.Body.Close()

     message = map[string]interface{}{
        "title":title,
     }
     requestBody,err = json.Marshal(message)
     if err != nil {
          log.Fatalln(err)
     } 
 
    req, err = http.NewRequest("PATCH", Walla.Url+"/api/entries/"+fmt.Sprint(id)+".json", bytes.NewBuffer(requestBody))

     req.Header.Set("Content-Type", "application/json")
     req.Header.Add("Authorization", bearer)
 
     client = &http.Client{}
     resp, err = client.Do(req)
     if err != nil {
         log.Fatalln(err)
     }
     defer resp.Body.Close()
 
}
 
 //Check if an article already exists in wallabag
 func WallaCheckArticleExists(Walla walla, url string) bool {
     
     bearer := "Bearer " + Walla.Token
 
     client := &http.Client{}
 
     req, err := http.NewRequest("GET", Walla.Url+"/api/entries/exists.json?url="+url, nil)
     req.Header.Add("Content-Type", "application/json")
     req.Header.Add("Authorization", bearer)
 
 
     resp, err := client.Do(req)   
     if err != nil {
         log.Fatalln(err)
     }
     var result map[string]interface{}
 
     json.NewDecoder(resp.Body).Decode(&result)
     defer resp.Body.Close()
     return result["exists"].(bool)
 }
 
 //Proccess rss and add new articles
 func ProcessRss(k sites, wg *sync.WaitGroup, Walla walla) {
     defer wg.Done()
     var exists bool
     client := &http.Client{Timeout: 25 * time.Second}
 
     resp, err := rss.ReadWithClient(k.Url,client, false)
     if err != nil {
         log.Println(err)
     }

     channel, err := rss.Regular(resp)
     for _, item := range channel.Item {
        exists = WallaCheckArticleExists(Walla, item.Link)
        //Mediapart fix
         if(!exists){
            exists = WallaCheckArticleExists(Walla, item.Link+"?onglet=full")    
        }
        if(exists) {
            log.Println(channel.Title+ " : " + item.Title + " Already exists skipping")
        } else {
            log.Println(channel.Title + " : " + "Adding " + item.Title)
            WallaAddUrl(Walla, item.Link, k.Tags, item.Title)
        }
     }
 }
 
 func main() {
 
     file, err := os.OpenFile("info.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
     if err != nil {
         log.Fatal(err)
     }
     defer file.Close()
     log.SetOutput(file)
 
     var MyWalla walla
     MyWalla.getWallaConf()
     MyWalla.Auth() 
    
     var c Sites
     c.getConf()
     log.Println(c)
     var wg sync.WaitGroup
     for k := range c {
         wg.Add(1)
        go  ProcessRss(c[k], &wg, MyWalla)
     }
 
     wg.Wait()
     log.Println("All done")
 }