From be76c08756500ccf363f6f4a61f65ab03d7528cc Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Mon, 14 Apr 2025 13:41:04 +0500 Subject: [PATCH 1/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=20Redis=20=D0=B2=20=D0=BA=D0=B0=D1=87=D0=B5=D1=81=D1=82=D0=B2?= =?UTF-8?q?=D0=B5=20=D1=81=D1=82=D0=B5=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 6 +- main.go | 256 ++++++----------------------------------- src/config/config.go | 63 ++++++++++ src/stacker/stacker.go | 43 +++++++ src/tooter/tooter.go | 170 +++++++++++++++++++++++++++ 5 files changed, 317 insertions(+), 221 deletions(-) create mode 100644 src/config/config.go create mode 100644 src/stacker/stacker.go create mode 100644 src/tooter/tooter.go diff --git a/go.mod b/go.mod index 4c8a8ee..094a8af 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,19 @@ go 1.23.0 toolchain go1.24.2 require ( - github.com/go-yaml/yaml v2.1.0+incompatible github.com/mattn/go-mastodon v0.0.9 github.com/mmcdole/gofeed v1.3.0 + github.com/redis/go-redis/v9 v9.7.3 github.com/urfave/cli/v3 v3.1.1 golang.org/x/net v0.39.0 + gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/PuerkitoBio/goquery v1.10.2 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mmcdole/goxpp v1.1.1 // indirect @@ -23,5 +26,4 @@ require ( github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect golang.org/x/text v0.24.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/main.go b/main.go index 6b00be9..49fee83 100644 --- a/main.go +++ b/main.go @@ -2,224 +2,20 @@ package main import ( "context" - "fmt" - "io" "log" - "net/http" "net/url" "os" "path/filepath" - "strings" "time" - "github.com/go-yaml/yaml" + "kiki/src/config" + "kiki/src/stacker" + "kiki/src/tooter" + "github.com/mattn/go-mastodon" - "github.com/mmcdole/gofeed" "github.com/urfave/cli/v3" - "golang.org/x/net/html" ) -type MastodonClientData struct { - ClientID string `yaml:"clientID,omitempty"` - ClientSecret string `yaml:"clientSecret,omitempty"` - AccessToken string `yaml:"accessToken,omitempty"` - Instance string `yaml:"instance,omitempty"` -} - -type KikiSettings struct { - Instance string `yaml:"instance,omitempty"` - RSSUri string `yaml:"rss_url,omitempty"` -} - -func getSecrets(path string) *mastodon.Config { - var clientData MastodonClientData - - secretConfig, err := os.ReadFile(path) - if err != nil { - log.Println(err) - } - - err = yaml.Unmarshal(secretConfig, &clientData) - if err != nil { - log.Println(err) - } - - config := &mastodon.Config{ - Server: clientData.Instance, - ClientID: clientData.ClientID, - ClientSecret: clientData.ClientSecret, - AccessToken: clientData.AccessToken, - } - - return config -} - -func getKikiConfig(path string) KikiSettings { - var kikiSettings KikiSettings - - kikiConfigFile, err := os.ReadFile(path) - if err != nil { - log.Println(err) - } - - err = yaml.Unmarshal(kikiConfigFile, &kikiSettings) - if err != nil { - log.Println(err) - } - - return kikiSettings -} - -func clientConfiguration(Instance string) { - appConfig := &mastodon.AppConfig{ - Server: Instance, - ClientName: "Kiki", - Scopes: "read write follow", - Website: "catgirls.asia", - RedirectURIs: "urn:ietf:wg:oauth:2.0:oob", - } - - app, err := mastodon.RegisterApp(context.Background(), appConfig) - if err != nil { - log.Println(err) - } - - u, err := url.Parse(app.AuthURI) - if err != nil { - log.Println(err) - } - var userToken string - fmt.Println(u) - fmt.Scanln(&userToken) - - config := &mastodon.Config{ - Server: Instance, - ClientID: app.ClientID, - ClientSecret: app.ClientSecret, - AccessToken: userToken, - } - - mastoClient := mastodon.NewClient(config) - err = mastoClient.AuthenticateToken(context.Background(), userToken, "urn:ietf:wg:oauth:2.0:oob") - if err != nil { - log.Println(err) - } - - clientData := MastodonClientData{ - Instance: Instance, - ClientID: mastoClient.Config.ClientID, - ClientSecret: mastoClient.Config.ClientSecret, - AccessToken: mastoClient.Config.AccessToken, - } - - marshaledYaml, err := yaml.Marshal(clientData) - if err != nil { - log.Println(err) - } - - log.Println(string(marshaledYaml)) - secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - if err != nil { - log.Println(err) - } - secretConfig.Write(marshaledYaml) - defer secretConfig.Close() -} - -func newsText(url string) []*gofeed.Item { - fp := gofeed.NewParser() - feed, err := fp.ParseURL(url) - if err != nil { - log.Println(err) - } - log.Println("RSS лента получена") - return feed.Items -} - -func createPost(mastoClient mastodon.Client, toot mastodon.Toot) { - _, err := mastoClient.PostStatus(context.Background(), &toot) - if err != nil { - log.Println(err) - } -} - -func picBytesArray(picturesArray []string) [][]byte { - var picturesBytes [][]byte - for _, picture := range picturesArray { - resp, err := http.Get(picture) - if err != nil { - log.Println(err) - return picturesBytes - } - defer resp.Body.Close() - picBytes, err := io.ReadAll(resp.Body) - if err != nil { - log.Println(err) - return picturesBytes - } - picturesBytes = append(picturesBytes, picBytes) - } - return picturesBytes -} - -func uploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment { - var attachments []*mastodon.Attachment - - for _, file := range filesBytes { - att, err := mastoClient.UploadMediaFromBytes(context.Background(), file) - - if err != nil { - log.Println(err) - return attachments - } - - attachments = append(attachments, att) - } - - return attachments -} - -func createToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) { - var tootText string - var imgArray []string - var attachments []*mastodon.Attachment - - toot := mastodon.Toot{ - Visibility: "unlisted", - Sensitive: true, - } - - uString := html.UnescapeString(newsDesc) - pHtml, err := html.Parse(strings.NewReader(uString)) - if err != nil { - return mastodon.Toot{}, err - } - - for n := range pHtml.Descendants() { - if n.Type != html.ElementNode { - tootText += (n.Data + "\n") - } - if n.Type == html.ElementNode && n.Data == "img" { - for _, attr := range n.Attr { - if attr.Key == "src" { - imgArray = append(imgArray, attr.Val) - } - } - } - } - - if len(imgArray) != 0 { - attachments = uploadPictures(mastoClient, picBytesArray(imgArray)) - } - - toot.Status = tootText - for _, attach := range attachments { - toot.MediaIDs = append(toot.MediaIDs, attach.ID) - } - - return toot, nil -} - func main() { cmd := &cli.Command{ Name: "kiki", @@ -230,7 +26,7 @@ func main() { Usage: "Инициализировать клиента", Action: func(ctx context.Context, cmd *cli.Command) error { confFile, err := filepath.Abs("config.yaml") - kikiConfig := getKikiConfig(confFile) + kikiConfig := config.GetKikiConfig(confFile) if err != nil { log.Println(err) @@ -242,7 +38,7 @@ func main() { } instanceUrlParser.Scheme = "https" - clientConfiguration(instanceUrlParser.String()) + tooter.ClientConfiguration(instanceUrlParser.String()) return nil }, @@ -251,30 +47,52 @@ func main() { Name: "run", Usage: "Запуск транслятора", Action: func(ctx context.Context, cmd *cli.Command) error { - var lastGUID string - mastoClient := mastodon.NewClient(getSecrets("secret.conf")) + mastoClient := mastodon.NewClient(config.GetSecrets("secret.conf")) confFile, err := filepath.Abs("config.yaml") if err != nil { log.Println(err) } - kikiConfig := getKikiConfig(confFile) + kikiConfig := config.GetKikiConfig(confFile) + + rdb := stacker.ConnectToRedis(kikiConfig.Redis.Address) ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() for range ticker.C { - news := newsText(kikiConfig.RSSUri) - if news[0].GUID != lastGUID { - log.Println(news[0].Description) - toot, err := createToot(*mastoClient, news[0].Description) + newPosts := tooter.NewsText(kikiConfig.RSSUri) + + for _, post := range newPosts { + inStack, err := stacker.CheckInRedis(rdb, post.GUID) if err != nil { log.Println(err) } - createPost(*mastoClient, toot) - lastGUID = news[0].GUID + if !inStack { + log.Println(post.Description) + + toot, err := tooter.CreateToot(*mastoClient, post.Description) + if err != nil { + log.Println(err) + } + + tooter.CreatePost(*mastoClient, toot) + + stacker.SetToRedis(rdb, post.GUID, post.Description) + } } + /*if news[0].GUID != lastGUID { + log.Println(news[0].Description) + + toot, err := tooter.CreateToot(*mastoClient, news[0].Description) + if err != nil { + log.Println(err) + } + + tooter.CreatePost(*mastoClient, toot) + lastGUID = news[0].GUID + }*/ } return nil }, diff --git a/src/config/config.go b/src/config/config.go new file mode 100644 index 0000000..d72955c --- /dev/null +++ b/src/config/config.go @@ -0,0 +1,63 @@ +package config + +import ( + "log" + "os" + + "github.com/mattn/go-mastodon" + "gopkg.in/yaml.v2" +) + +type KikiSettings struct { + Instance string `yaml:"instance,omitempty"` + RSSUri string `yaml:"rss_url,omitempty"` + Redis struct { + Address string `yaml:"address"` + } `yaml:"redis"` +} + +type MastodonClientData struct { + ClientID string `yaml:"clientID,omitempty"` + ClientSecret string `yaml:"clientSecret,omitempty"` + AccessToken string `yaml:"accessToken,omitempty"` + Instance string `yaml:"instance,omitempty"` +} + +func GetKikiConfig(path string) KikiSettings { + var kikiSettings KikiSettings + + kikiConfigFile, err := os.ReadFile(path) + if err != nil { + log.Println(err) + } + + err = yaml.Unmarshal(kikiConfigFile, &kikiSettings) + if err != nil { + log.Println(err) + } + + return kikiSettings +} + +func GetSecrets(path string) *mastodon.Config { + var clientData MastodonClientData + + secretConfig, err := os.ReadFile(path) + if err != nil { + log.Println(err) + } + + err = yaml.Unmarshal(secretConfig, &clientData) + if err != nil { + log.Println(err) + } + + config := &mastodon.Config{ + Server: clientData.Instance, + ClientID: clientData.ClientID, + ClientSecret: clientData.ClientSecret, + AccessToken: clientData.AccessToken, + } + + return config +} diff --git a/src/stacker/stacker.go b/src/stacker/stacker.go new file mode 100644 index 0000000..8f3de73 --- /dev/null +++ b/src/stacker/stacker.go @@ -0,0 +1,43 @@ +package stacker + +import ( + "context" + + "github.com/redis/go-redis/v9" +) + +func ConnectToRedis(addr string) redis.Client { + rdb := redis.NewClient(&redis.Options{ + Addr: addr, + Password: "", + DB: 0, + }) + + return *rdb +} + +func SetToRedis(rdb redis.Client, key string, val interface{}) error { + err := rdb.Set(context.Background(), key, val, 0).Err() + if err != nil { + return err + } + return nil +} + +func CheckInRedis(rdb redis.Client, key string) (bool, error) { + _, err := rdb.Get(context.Background(), key).Result() + if err == redis.Nil { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +func SaveRedis(rdb redis.Client) error { + err := rdb.Save(context.Background()).Err() + if err != nil { + return err + } + return nil +} diff --git a/src/tooter/tooter.go b/src/tooter/tooter.go new file mode 100644 index 0000000..3f1d762 --- /dev/null +++ b/src/tooter/tooter.go @@ -0,0 +1,170 @@ +package tooter + +import ( + "context" + "fmt" + "io" + "kiki/src/config" + "log" + "net/http" + "net/url" + "os" + "strings" + + "github.com/mattn/go-mastodon" + "github.com/mmcdole/gofeed" + "golang.org/x/net/html" + "gopkg.in/yaml.v2" +) + +func ClientConfiguration(Instance string) { + appConfig := &mastodon.AppConfig{ + Server: Instance, + ClientName: "Kiki", + Scopes: "read write follow", + Website: "catgirls.asia", + RedirectURIs: "urn:ietf:wg:oauth:2.0:oob", + } + + app, err := mastodon.RegisterApp(context.Background(), appConfig) + if err != nil { + log.Println(err) + } + + u, err := url.Parse(app.AuthURI) + if err != nil { + log.Println(err) + } + var userToken string + fmt.Println(u) + fmt.Scanln(&userToken) + + conf := &mastodon.Config{ + Server: Instance, + ClientID: app.ClientID, + ClientSecret: app.ClientSecret, + AccessToken: userToken, + } + + mastoClient := mastodon.NewClient(conf) + err = mastoClient.AuthenticateToken(context.Background(), userToken, "urn:ietf:wg:oauth:2.0:oob") + if err != nil { + log.Println(err) + } + + clientData := config.MastodonClientData{ + Instance: Instance, + ClientID: mastoClient.Config.ClientID, + ClientSecret: mastoClient.Config.ClientSecret, + AccessToken: mastoClient.Config.AccessToken, + } + + marshaledYaml, err := yaml.Marshal(clientData) + if err != nil { + log.Println(err) + } + + log.Println(string(marshaledYaml)) + secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + log.Println(err) + } + secretConfig.Write(marshaledYaml) + defer secretConfig.Close() +} + +func NewsText(url string) []*gofeed.Item { + fp := gofeed.NewParser() + feed, err := fp.ParseURL(url) + if err != nil { + log.Println(err) + } + log.Println("RSS лента получена") + return feed.Items +} + +func CreatePost(mastoClient mastodon.Client, toot mastodon.Toot) { + _, err := mastoClient.PostStatus(context.Background(), &toot) + if err != nil { + log.Println(err) + } +} + +func PicBytesArray(picturesArray []string) [][]byte { + var picturesBytes [][]byte + for _, picture := range picturesArray { + resp, err := http.Get(picture) + if err != nil { + log.Println(err) + return picturesBytes + } + defer resp.Body.Close() + picBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Println(err) + return picturesBytes + } + picturesBytes = append(picturesBytes, picBytes) + } + return picturesBytes +} + +func UploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment { + var attachments []*mastodon.Attachment + + for _, file := range filesBytes { + att, err := mastoClient.UploadMediaFromBytes(context.Background(), file) + + if err != nil { + log.Println(err) + return attachments + } + + attachments = append(attachments, att) + } + + return attachments +} + +func CreateToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) { + var tootText string + var imgArray []string + var attachments []*mastodon.Attachment + + toot := mastodon.Toot{ + Visibility: "unlisted", + Sensitive: true, + } + + uString := html.UnescapeString(newsDesc) + pHtml, err := html.Parse(strings.NewReader(uString)) + if err != nil { + return mastodon.Toot{}, err + } + + for n := range pHtml.Descendants() { + if n.Type != html.ElementNode { + tootText += (n.Data + "\n") + } + + if n.Type == html.ElementNode && n.Data == "img" { + for _, attr := range n.Attr { + if attr.Key == "src" { + imgArray = append(imgArray, attr.Val) + } + } + } + + } + + if len(imgArray) != 0 { + attachments = UploadPictures(mastoClient, PicBytesArray(imgArray)) + } + + toot.Status = tootText + for _, attach := range attachments { + toot.MediaIDs = append(toot.MediaIDs, attach.ID) + } + + return toot, nil +} From c853cfb592b9adc7e7f2b4e89a35073d360c0ffa Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Mon, 21 Apr 2025 23:13:46 +0500 Subject: [PATCH 2/7] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=B2=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D1=88=D0=B8=D0=BB=20=D1=84=D0=B0=D0=B9=D0=BB=D1=8B=20(?= =?UTF-8?q?=D0=BE=D0=BF=D1=8F=D1=82=D1=8C),=20=D0=B7=D0=B0=D0=BA=D0=B8?= =?UTF-8?q?=D0=BD=D1=83=D0=BB=20=D0=B2=D1=81=D1=91=20=D0=B2=20=D0=B4=D0=BE?= =?UTF-8?q?=D0=BA=D0=B5=D1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 16 +++++++++++++ {src/config => config}/config.go | 0 config/go.mod | 14 ++++++++++++ docker-compose.yml | 11 +++++++++ go.mod | 29 ------------------------ service/go.mod | 35 +++++++++++++++++++++++++++++ main.go => service/kiki.go | 20 +++++------------ stacker/go.mod | 10 +++++++++ {src/stacker => stacker}/stacker.go | 0 tooter/go.mod | 25 +++++++++++++++++++++ {src/tooter => tooter}/tooter.go | 29 +++++++++++++++--------- 11 files changed, 135 insertions(+), 54 deletions(-) create mode 100644 Dockerfile rename {src/config => config}/config.go (100%) create mode 100644 config/go.mod create mode 100644 docker-compose.yml delete mode 100644 go.mod create mode 100644 service/go.mod rename main.go => service/kiki.go (82%) create mode 100644 stacker/go.mod rename {src/stacker => stacker}/stacker.go (100%) create mode 100644 tooter/go.mod rename {src/tooter => tooter}/tooter.go (84%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a6020e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24 + +WORKDIR /app/kiki + +RUN mkdir /app/kiki/config /app/kiki/service /app/kiki/stacker /app/kiki/tooter + +COPY config/* /app/kiki/config +COPY service/* /app/kiki/service +COPY stacker/* /app/kiki/stacker +COPY tooter/* /app/kiki/tooter + +RUN cd service && go mod tidy && cd ../ + +RUN go build -C ./service -o ../kiki + +CMD [ "/app/kiki/kiki", "run" ] \ No newline at end of file diff --git a/src/config/config.go b/config/config.go similarity index 100% rename from src/config/config.go rename to config/config.go diff --git a/config/go.mod b/config/go.mod new file mode 100644 index 0000000..390a727 --- /dev/null +++ b/config/go.mod @@ -0,0 +1,14 @@ +module kiki/config + +go 1.24.2 + +require ( + github.com/mattn/go-mastodon v0.0.9 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/gorilla/websocket v1.5.1 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + golang.org/x/net v0.25.0 // indirect +) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8d17708 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + kiki: + build: ./ + restart: always + volumes: + - ./config.yaml:/app/kiki/config.yaml + - ./secret.conf:/app/kiki/secret.conf + redis: + image: redis + restart: always + \ No newline at end of file diff --git a/go.mod b/go.mod deleted file mode 100644 index 094a8af..0000000 --- a/go.mod +++ /dev/null @@ -1,29 +0,0 @@ -module kiki - -go 1.23.0 - -toolchain go1.24.2 - -require ( - github.com/mattn/go-mastodon v0.0.9 - github.com/mmcdole/gofeed v1.3.0 - github.com/redis/go-redis/v9 v9.7.3 - github.com/urfave/cli/v3 v3.1.1 - golang.org/x/net v0.39.0 - gopkg.in/yaml.v2 v2.4.0 -) - -require ( - github.com/PuerkitoBio/goquery v1.10.2 // indirect - github.com/andybalholm/cascadia v1.3.3 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/gorilla/websocket v1.5.3 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mmcdole/goxpp v1.1.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect - golang.org/x/text v0.24.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect -) diff --git a/service/go.mod b/service/go.mod new file mode 100644 index 0000000..7f97ff1 --- /dev/null +++ b/service/go.mod @@ -0,0 +1,35 @@ +module kiki/kiki + +go 1.24.2 + +replace kiki/config => ../config + +replace kiki/stacker => ../stacker + +replace kiki/tooter => ../tooter + +require ( + github.com/mattn/go-mastodon v0.0.9 + github.com/urfave/cli/v3 v3.2.0 + kiki/config v0.0.0-00010101000000-000000000000 + kiki/stacker v0.0.0-00010101000000-000000000000 + kiki/tooter v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/gofeed v1.3.0 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/redis/go-redis/v9 v9.7.3 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/text v0.24.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/main.go b/service/kiki.go similarity index 82% rename from main.go rename to service/kiki.go index 49fee83..8698431 100644 --- a/main.go +++ b/service/kiki.go @@ -8,9 +8,9 @@ import ( "path/filepath" "time" - "kiki/src/config" - "kiki/src/stacker" - "kiki/src/tooter" + "kiki/config" + "kiki/stacker" + "kiki/tooter" "github.com/mattn/go-mastodon" "github.com/urfave/cli/v3" @@ -56,6 +56,7 @@ func main() { kikiConfig := config.GetKikiConfig(confFile) rdb := stacker.ConnectToRedis(kikiConfig.Redis.Address) + defer stacker.SaveRedis(rdb) ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() @@ -72,7 +73,7 @@ func main() { if !inStack { log.Println(post.Description) - toot, err := tooter.CreateToot(*mastoClient, post.Description) + toot, err := tooter.CreateToot(*mastoClient, post) if err != nil { log.Println(err) } @@ -82,17 +83,6 @@ func main() { stacker.SetToRedis(rdb, post.GUID, post.Description) } } - /*if news[0].GUID != lastGUID { - log.Println(news[0].Description) - - toot, err := tooter.CreateToot(*mastoClient, news[0].Description) - if err != nil { - log.Println(err) - } - - tooter.CreatePost(*mastoClient, toot) - lastGUID = news[0].GUID - }*/ } return nil }, diff --git a/stacker/go.mod b/stacker/go.mod new file mode 100644 index 0000000..4b1afb2 --- /dev/null +++ b/stacker/go.mod @@ -0,0 +1,10 @@ +module kiki/stacker + +go 1.24.2 + +require github.com/redis/go-redis/v9 v9.7.3 + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect +) diff --git a/src/stacker/stacker.go b/stacker/stacker.go similarity index 100% rename from src/stacker/stacker.go rename to stacker/stacker.go diff --git a/tooter/go.mod b/tooter/go.mod new file mode 100644 index 0000000..27e1817 --- /dev/null +++ b/tooter/go.mod @@ -0,0 +1,25 @@ +module kiki/tooter + +go 1.24.2 + +require ( + github.com/mattn/go-mastodon v0.0.9 + github.com/mmcdole/gofeed v1.3.0 + golang.org/x/net v0.39.0 + gopkg.in/yaml.v2 v2.4.0 + kiki/config v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + golang.org/x/text v0.24.0 // indirect +) + +replace kiki/config => ../config diff --git a/src/tooter/tooter.go b/tooter/tooter.go similarity index 84% rename from src/tooter/tooter.go rename to tooter/tooter.go index 3f1d762..7dd91a8 100644 --- a/src/tooter/tooter.go +++ b/tooter/tooter.go @@ -4,13 +4,14 @@ import ( "context" "fmt" "io" - "kiki/src/config" "log" "net/http" "net/url" "os" "strings" + "kiki/config" + "github.com/mattn/go-mastodon" "github.com/mmcdole/gofeed" "golang.org/x/net/html" @@ -36,7 +37,8 @@ func ClientConfiguration(Instance string) { log.Println(err) } var userToken string - fmt.Println(u) + //fmt.Println(u) + fmt.Printf("Перейдите по ссылке\n%s\nИ введите user token ниже:\n", u) fmt.Scanln(&userToken) conf := &mastodon.Config{ @@ -126,17 +128,18 @@ func UploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodo return attachments } -func CreateToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) { - var tootText string +func CreateToot(mastoClient mastodon.Client, newsDesc *gofeed.Item) (mastodon.Toot, error) { var imgArray []string var attachments []*mastodon.Attachment + var tootText string = fmt.Sprintf("src: %s\n\n", newsDesc.Link) + toot := mastodon.Toot{ Visibility: "unlisted", Sensitive: true, } - uString := html.UnescapeString(newsDesc) + uString := html.UnescapeString(newsDesc.Description) pHtml, err := html.Parse(strings.NewReader(uString)) if err != nil { return mastodon.Toot{}, err @@ -144,14 +147,20 @@ func CreateToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, er for n := range pHtml.Descendants() { if n.Type != html.ElementNode { - tootText += (n.Data + "\n") + tootText += (n.Data) } - if n.Type == html.ElementNode && n.Data == "img" { - for _, attr := range n.Attr { - if attr.Key == "src" { - imgArray = append(imgArray, attr.Val) + if n.Type == html.ElementNode { + switch n.Data { + case "img": + for _, attr := range n.Attr { + if attr.Key == "src" { + imgArray = append(imgArray, attr.Val) + } } + + case "br": + tootText += "\n" } } From 1f1c964e917905a3811bfc1e35e3275f8d0ea08e Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Mon, 21 Apr 2025 23:22:25 +0500 Subject: [PATCH 3/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=B7=D0=B0=D0=B2=D0=B8=D1=81=D0=B8=D0=BC=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 8d17708..3e8a0af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: volumes: - ./config.yaml:/app/kiki/config.yaml - ./secret.conf:/app/kiki/secret.conf + depends_on: + - redis redis: image: redis restart: always From fef2d2fdb6f471105aab287a38dd630f6d973568 Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Wed, 23 Apr 2025 09:09:01 +0500 Subject: [PATCH 4/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=B2=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=BD=D0=B8=D0=B5/?= =?UTF-8?q?=D0=BE=D1=82=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?NSFW=20=D1=88=D1=82=D1=83=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- config/config.go | 7 ++++--- service/kiki.go | 2 +- tooter/tooter.go | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 83d7ac2..867ccf5 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ config.yaml # Output of the go coverage tool, specifically when used with LiteIDE *.out - +kiki # Dependency directories (remove the comment below to include it) # vendor/ diff --git a/config/config.go b/config/config.go index d72955c..b84cd21 100644 --- a/config/config.go +++ b/config/config.go @@ -9,9 +9,10 @@ import ( ) type KikiSettings struct { - Instance string `yaml:"instance,omitempty"` - RSSUri string `yaml:"rss_url,omitempty"` - Redis struct { + Instance string `yaml:"instance,omitempty"` + RSSUri string `yaml:"rss_url,omitempty"` + Sensitive bool `yaml:"sensitive,omitempty"` + Redis struct { Address string `yaml:"address"` } `yaml:"redis"` } diff --git a/service/kiki.go b/service/kiki.go index 8698431..420c0dd 100644 --- a/service/kiki.go +++ b/service/kiki.go @@ -73,7 +73,7 @@ func main() { if !inStack { log.Println(post.Description) - toot, err := tooter.CreateToot(*mastoClient, post) + toot, err := tooter.CreateToot(*mastoClient, post, kikiConfig.Sensitive) if err != nil { log.Println(err) } diff --git a/tooter/tooter.go b/tooter/tooter.go index 7dd91a8..871c0cc 100644 --- a/tooter/tooter.go +++ b/tooter/tooter.go @@ -128,7 +128,7 @@ func UploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodo return attachments } -func CreateToot(mastoClient mastodon.Client, newsDesc *gofeed.Item) (mastodon.Toot, error) { +func CreateToot(mastoClient mastodon.Client, newsDesc *gofeed.Item, sensitive bool) (mastodon.Toot, error) { var imgArray []string var attachments []*mastodon.Attachment From 664bc8df763ed8ec6fbc23f6a58b6f80b372f3ec Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Wed, 23 Apr 2025 09:10:18 +0500 Subject: [PATCH 5/7] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=BF=D1=80=D0=B8=D0=BC=D0=B5=D1=80=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D1=84=D0=B8=D0=B3=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.example.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config.example.yaml b/config.example.yaml index 3b02854..10cff5e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,2 +1,5 @@ instance: https://pleroma.catgirls.asia -rss_url: https://4pda.to/feed \ No newline at end of file +rss_url: https://4pda.to/feed +sensitive: true +redis: + address: localhost:6379 \ No newline at end of file From 4d3833850687f282a6f7b4a0e685115c023d0216 Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Wed, 23 Apr 2025 09:19:25 +0500 Subject: [PATCH 6/7] =?UTF-8?q?=D0=A2=D0=B5=D0=BF=D0=B5=D1=80=D1=8C=20?= =?UTF-8?q?=D1=82=D1=8D=D0=B3=20

=20=D1=82=D0=BE=D0=B6=D0=B5=20=D0=B4?= =?UTF-8?q?=D0=B5=D0=BB=D0=B0=D0=B5=D1=82=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B2?= =?UTF-8?q?=D0=BE=D0=B4=20=D0=BD=D0=B0=20=D0=BD=D0=BE=D0=B2=D1=83=D1=8E=20?= =?UTF-8?q?=D1=81=D1=82=D1=80=D0=BE=D0=BA=D1=83...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tooter/tooter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooter/tooter.go b/tooter/tooter.go index 871c0cc..71ec54a 100644 --- a/tooter/tooter.go +++ b/tooter/tooter.go @@ -159,7 +159,7 @@ func CreateToot(mastoClient mastodon.Client, newsDesc *gofeed.Item, sensitive bo } } - case "br": + case "br", "p": tootText += "\n" } } From 10474bb165c55d05ba04e22669e2ac54b564e539 Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Wed, 23 Apr 2025 09:57:44 +0500 Subject: [PATCH 7/7] =?UTF-8?q?=D0=97=D0=B0=D0=B1=D1=8B=D0=BB=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82=D1=8C=20=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=BC=D0=B5=D0=BD=D0=BD=D1=83=D1=8E=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20NSFW...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tooter/tooter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooter/tooter.go b/tooter/tooter.go index 71ec54a..69fbb60 100644 --- a/tooter/tooter.go +++ b/tooter/tooter.go @@ -136,7 +136,7 @@ func CreateToot(mastoClient mastodon.Client, newsDesc *gofeed.Item, sensitive bo toot := mastodon.Toot{ Visibility: "unlisted", - Sensitive: true, + Sensitive: sensitive, } uString := html.UnescapeString(newsDesc.Description)