Добавил Redis в качестве стека

go.mod

@@ -5,16 +5,19 @@ go 1.23.0
 toolchain go1.24.2
 
 require (
-	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/mattn/go-mastodon v0.0.9
 	github.com/mmcdole/gofeed v1.3.0
+	github.com/redis/go-redis/v9 v9.7.3
 	github.com/urfave/cli/v3 v3.1.1
 	golang.org/x/net v0.39.0
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/PuerkitoBio/goquery v1.10.2 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mmcdole/goxpp v1.1.1 // indirect
@@ -23,5 +26,4 @@ require (
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	golang.org/x/text v0.24.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

main.go

@@ -2,224 +2,20 @@ package main
 
 import (
 	"context"
-	"fmt"
-	"io"
 	"log"
-	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
-	"github.com/go-yaml/yaml"
+	"kiki/src/config"
+	"kiki/src/stacker"
+	"kiki/src/tooter"
+
 	"github.com/mattn/go-mastodon"
-	"github.com/mmcdole/gofeed"
 	"github.com/urfave/cli/v3"
-	"golang.org/x/net/html"
 )
 
-type MastodonClientData struct {
-	ClientID     string `yaml:"clientID,omitempty"`
-	ClientSecret string `yaml:"clientSecret,omitempty"`
-	AccessToken  string `yaml:"accessToken,omitempty"`
-	Instance     string `yaml:"instance,omitempty"`
-}
-
-type KikiSettings struct {
-	Instance string `yaml:"instance,omitempty"`
-	RSSUri   string `yaml:"rss_url,omitempty"`
-}
-
-func getSecrets(path string) *mastodon.Config {
-	var clientData MastodonClientData
-
-	secretConfig, err := os.ReadFile(path)
-	if err != nil {
-		log.Println(err)
-	}
-
-	err = yaml.Unmarshal(secretConfig, &clientData)
-	if err != nil {
-		log.Println(err)
-	}
-
-	config := &mastodon.Config{
-		Server:       clientData.Instance,
-		ClientID:     clientData.ClientID,
-		ClientSecret: clientData.ClientSecret,
-		AccessToken:  clientData.AccessToken,
-	}
-
-	return config
-}
-
-func getKikiConfig(path string) KikiSettings {
-	var kikiSettings KikiSettings
-
-	kikiConfigFile, err := os.ReadFile(path)
-	if err != nil {
-		log.Println(err)
-	}
-
-	err = yaml.Unmarshal(kikiConfigFile, &kikiSettings)
-	if err != nil {
-		log.Println(err)
-	}
-
-	return kikiSettings
-}
-
-func clientConfiguration(Instance string) {
-	appConfig := &mastodon.AppConfig{
-		Server:       Instance,
-		ClientName:   "Kiki",
-		Scopes:       "read write follow",
-		Website:      "catgirls.asia",
-		RedirectURIs: "urn:ietf:wg:oauth:2.0:oob",
-	}
-
-	app, err := mastodon.RegisterApp(context.Background(), appConfig)
-	if err != nil {
-		log.Println(err)
-	}
-
-	u, err := url.Parse(app.AuthURI)
-	if err != nil {
-		log.Println(err)
-	}
-	var userToken string
-	fmt.Println(u)
-	fmt.Scanln(&userToken)
-
-	config := &mastodon.Config{
-		Server:       Instance,
-		ClientID:     app.ClientID,
-		ClientSecret: app.ClientSecret,
-		AccessToken:  userToken,
-	}
-
-	mastoClient := mastodon.NewClient(config)
-	err = mastoClient.AuthenticateToken(context.Background(), userToken, "urn:ietf:wg:oauth:2.0:oob")
-	if err != nil {
-		log.Println(err)
-	}
-
-	clientData := MastodonClientData{
-		Instance:     Instance,
-		ClientID:     mastoClient.Config.ClientID,
-		ClientSecret: mastoClient.Config.ClientSecret,
-		AccessToken:  mastoClient.Config.AccessToken,
-	}
-
-	marshaledYaml, err := yaml.Marshal(clientData)
-	if err != nil {
-		log.Println(err)
-	}
-
-	log.Println(string(marshaledYaml))
-	secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
-	if err != nil {
-		log.Println(err)
-	}
-	secretConfig.Write(marshaledYaml)
-	defer secretConfig.Close()
-}
-
-func newsText(url string) []*gofeed.Item {
-	fp := gofeed.NewParser()
-	feed, err := fp.ParseURL(url)
-	if err != nil {
-		log.Println(err)
-	}
-	log.Println("RSS лента получена")
-	return feed.Items
-}
-
-func createPost(mastoClient mastodon.Client, toot mastodon.Toot) {
-	_, err := mastoClient.PostStatus(context.Background(), &toot)
-	if err != nil {
-		log.Println(err)
-	}
-}
-
-func picBytesArray(picturesArray []string) [][]byte {
-	var picturesBytes [][]byte
-	for _, picture := range picturesArray {
-		resp, err := http.Get(picture)
-		if err != nil {
-			log.Println(err)
-			return picturesBytes
-		}
-		defer resp.Body.Close()
-		picBytes, err := io.ReadAll(resp.Body)
-		if err != nil {
-			log.Println(err)
-			return picturesBytes
-		}
-		picturesBytes = append(picturesBytes, picBytes)
-	}
-	return picturesBytes
-}
-
-func uploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment {
-	var attachments []*mastodon.Attachment
-
-	for _, file := range filesBytes {
-		att, err := mastoClient.UploadMediaFromBytes(context.Background(), file)
-
-		if err != nil {
-			log.Println(err)
-			return attachments
-		}
-
-		attachments = append(attachments, att)
-	}
-
-	return attachments
-}
-
-func createToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) {
-	var tootText string
-	var imgArray []string
-	var attachments []*mastodon.Attachment
-
-	toot := mastodon.Toot{
-		Visibility: "unlisted",
-		Sensitive:  true,
-	}
-
-	uString := html.UnescapeString(newsDesc)
-	pHtml, err := html.Parse(strings.NewReader(uString))
-	if err != nil {
-		return mastodon.Toot{}, err
-	}
-
-	for n := range pHtml.Descendants() {
-		if n.Type != html.ElementNode {
-			tootText += (n.Data + "\n")
-		}
-		if n.Type == html.ElementNode && n.Data == "img" {
-			for _, attr := range n.Attr {
-				if attr.Key == "src" {
-					imgArray = append(imgArray, attr.Val)
-				}
-			}
-		}
-	}
-
-	if len(imgArray) != 0 {
-		attachments = uploadPictures(mastoClient, picBytesArray(imgArray))
-	}
-
-	toot.Status = tootText
-	for _, attach := range attachments {
-		toot.MediaIDs = append(toot.MediaIDs, attach.ID)
-	}
-
-	return toot, nil
-}
-
 func main() {
 	cmd := &cli.Command{
 		Name:  "kiki",
@@ -230,7 +26,7 @@ func main() {
 				Usage: "Инициализировать клиента",
 				Action: func(ctx context.Context, cmd *cli.Command) error {
 					confFile, err := filepath.Abs("config.yaml")
-					kikiConfig := getKikiConfig(confFile)
+					kikiConfig := config.GetKikiConfig(confFile)
 
 					if err != nil {
 						log.Println(err)
@@ -242,7 +38,7 @@ func main() {
 					}
 					instanceUrlParser.Scheme = "https"
 
-					clientConfiguration(instanceUrlParser.String())
+					tooter.ClientConfiguration(instanceUrlParser.String())
 
 					return nil
 				},
@@ -251,30 +47,52 @@ func main() {
 				Name:  "run",
 				Usage: "Запуск транслятора",
 				Action: func(ctx context.Context, cmd *cli.Command) error {
-					var lastGUID string
+					mastoClient := mastodon.NewClient(config.GetSecrets("secret.conf"))
-					mastoClient := mastodon.NewClient(getSecrets("secret.conf"))
 
 					confFile, err := filepath.Abs("config.yaml")
 					if err != nil {
 						log.Println(err)
 					}
-					kikiConfig := getKikiConfig(confFile)
+					kikiConfig := config.GetKikiConfig(confFile)
+
+					rdb := stacker.ConnectToRedis(kikiConfig.Redis.Address)
 
 					ticker := time.NewTicker(1 * time.Minute)
 					defer ticker.Stop()
 					for range ticker.C {
-						news := newsText(kikiConfig.RSSUri)
-						if news[0].GUID != lastGUID {
-							log.Println(news[0].Description)
 
-							toot, err := createToot(*mastoClient, news[0].Description)
+						newPosts := tooter.NewsText(kikiConfig.RSSUri)
+
+						for _, post := range newPosts {
+							inStack, err := stacker.CheckInRedis(rdb, post.GUID)
 							if err != nil {
 								log.Println(err)
 							}
 
-							createPost(*mastoClient, toot)
+							if !inStack {
-							lastGUID = news[0].GUID
+								log.Println(post.Description)
+
+								toot, err := tooter.CreateToot(*mastoClient, post.Description)
+								if err != nil {
+									log.Println(err)
 								}
+
+								tooter.CreatePost(*mastoClient, toot)
+
+								stacker.SetToRedis(rdb, post.GUID, post.Description)
+							}
+						}
+						/*if news[0].GUID != lastGUID {
+							log.Println(news[0].Description)
+
+							toot, err := tooter.CreateToot(*mastoClient, news[0].Description)
+							if err != nil {
+								log.Println(err)
+							}
+
+							tooter.CreatePost(*mastoClient, toot)
+							lastGUID = news[0].GUID
+						}*/
 					}
 					return nil
 				},

src/config/config.go

@@ -0,0 +1,63 @@
+package config
+
+import (
+	"log"
+	"os"
+
+	"github.com/mattn/go-mastodon"
+	"gopkg.in/yaml.v2"
+)
+
+type KikiSettings struct {
+	Instance string `yaml:"instance,omitempty"`
+	RSSUri   string `yaml:"rss_url,omitempty"`
+	Redis    struct {
+		Address string `yaml:"address"`
+	} `yaml:"redis"`
+}
+
+type MastodonClientData struct {
+	ClientID     string `yaml:"clientID,omitempty"`
+	ClientSecret string `yaml:"clientSecret,omitempty"`
+	AccessToken  string `yaml:"accessToken,omitempty"`
+	Instance     string `yaml:"instance,omitempty"`
+}
+
+func GetKikiConfig(path string) KikiSettings {
+	var kikiSettings KikiSettings
+
+	kikiConfigFile, err := os.ReadFile(path)
+	if err != nil {
+		log.Println(err)
+	}
+
+	err = yaml.Unmarshal(kikiConfigFile, &kikiSettings)
+	if err != nil {
+		log.Println(err)
+	}
+
+	return kikiSettings
+}
+
+func GetSecrets(path string) *mastodon.Config {
+	var clientData MastodonClientData
+
+	secretConfig, err := os.ReadFile(path)
+	if err != nil {
+		log.Println(err)
+	}
+
+	err = yaml.Unmarshal(secretConfig, &clientData)
+	if err != nil {
+		log.Println(err)
+	}
+
+	config := &mastodon.Config{
+		Server:       clientData.Instance,
+		ClientID:     clientData.ClientID,
+		ClientSecret: clientData.ClientSecret,
+		AccessToken:  clientData.AccessToken,
+	}
+
+	return config
+}

src/stacker/stacker.go

@@ -0,0 +1,43 @@
+package stacker
+
+import (
+	"context"
+
+	"github.com/redis/go-redis/v9"
+)
+
+func ConnectToRedis(addr string) redis.Client {
+	rdb := redis.NewClient(&redis.Options{
+		Addr:     addr,
+		Password: "",
+		DB:       0,
+	})
+
+	return *rdb
+}
+
+func SetToRedis(rdb redis.Client, key string, val interface{}) error {
+	err := rdb.Set(context.Background(), key, val, 0).Err()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func CheckInRedis(rdb redis.Client, key string) (bool, error) {
+	_, err := rdb.Get(context.Background(), key).Result()
+	if err == redis.Nil {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func SaveRedis(rdb redis.Client) error {
+	err := rdb.Save(context.Background()).Err()
+	if err != nil {
+		return err
+	}
+	return nil
+}

src/tooter/tooter.go

@@ -0,0 +1,170 @@
+package tooter
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"kiki/src/config"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-mastodon"
+	"github.com/mmcdole/gofeed"
+	"golang.org/x/net/html"
+	"gopkg.in/yaml.v2"
+)
+
+func ClientConfiguration(Instance string) {
+	appConfig := &mastodon.AppConfig{
+		Server:       Instance,
+		ClientName:   "Kiki",
+		Scopes:       "read write follow",
+		Website:      "catgirls.asia",
+		RedirectURIs: "urn:ietf:wg:oauth:2.0:oob",
+	}
+
+	app, err := mastodon.RegisterApp(context.Background(), appConfig)
+	if err != nil {
+		log.Println(err)
+	}
+
+	u, err := url.Parse(app.AuthURI)
+	if err != nil {
+		log.Println(err)
+	}
+	var userToken string
+	fmt.Println(u)
+	fmt.Scanln(&userToken)
+
+	conf := &mastodon.Config{
+		Server:       Instance,
+		ClientID:     app.ClientID,
+		ClientSecret: app.ClientSecret,
+		AccessToken:  userToken,
+	}
+
+	mastoClient := mastodon.NewClient(conf)
+	err = mastoClient.AuthenticateToken(context.Background(), userToken, "urn:ietf:wg:oauth:2.0:oob")
+	if err != nil {
+		log.Println(err)
+	}
+
+	clientData := config.MastodonClientData{
+		Instance:     Instance,
+		ClientID:     mastoClient.Config.ClientID,
+		ClientSecret: mastoClient.Config.ClientSecret,
+		AccessToken:  mastoClient.Config.AccessToken,
+	}
+
+	marshaledYaml, err := yaml.Marshal(clientData)
+	if err != nil {
+		log.Println(err)
+	}
+
+	log.Println(string(marshaledYaml))
+	secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
+	if err != nil {
+		log.Println(err)
+	}
+	secretConfig.Write(marshaledYaml)
+	defer secretConfig.Close()
+}
+
+func NewsText(url string) []*gofeed.Item {
+	fp := gofeed.NewParser()
+	feed, err := fp.ParseURL(url)
+	if err != nil {
+		log.Println(err)
+	}
+	log.Println("RSS лента получена")
+	return feed.Items
+}
+
+func CreatePost(mastoClient mastodon.Client, toot mastodon.Toot) {
+	_, err := mastoClient.PostStatus(context.Background(), &toot)
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+func PicBytesArray(picturesArray []string) [][]byte {
+	var picturesBytes [][]byte
+	for _, picture := range picturesArray {
+		resp, err := http.Get(picture)
+		if err != nil {
+			log.Println(err)
+			return picturesBytes
+		}
+		defer resp.Body.Close()
+		picBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			log.Println(err)
+			return picturesBytes
+		}
+		picturesBytes = append(picturesBytes, picBytes)
+	}
+	return picturesBytes
+}
+
+func UploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment {
+	var attachments []*mastodon.Attachment
+
+	for _, file := range filesBytes {
+		att, err := mastoClient.UploadMediaFromBytes(context.Background(), file)
+
+		if err != nil {
+			log.Println(err)
+			return attachments
+		}
+
+		attachments = append(attachments, att)
+	}
+
+	return attachments
+}
+
+func CreateToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) {
+	var tootText string
+	var imgArray []string
+	var attachments []*mastodon.Attachment
+
+	toot := mastodon.Toot{
+		Visibility: "unlisted",
+		Sensitive:  true,
+	}
+
+	uString := html.UnescapeString(newsDesc)
+	pHtml, err := html.Parse(strings.NewReader(uString))
+	if err != nil {
+		return mastodon.Toot{}, err
+	}
+
+	for n := range pHtml.Descendants() {
+		if n.Type != html.ElementNode {
+			tootText += (n.Data + "\n")
+		}
+
+		if n.Type == html.ElementNode && n.Data == "img" {
+			for _, attr := range n.Attr {
+				if attr.Key == "src" {
+					imgArray = append(imgArray, attr.Val)
+				}
+			}
+		}
+
+	}
+
+	if len(imgArray) != 0 {
+		attachments = UploadPictures(mastoClient, PicBytesArray(imgArray))
+	}
+
+	toot.Status = tootText
+	for _, attach := range attachments {
+		toot.MediaIDs = append(toot.MediaIDs, attach.ID)
+	}
+
+	return toot, nil
+}