From 9d3ed1cd8d25f0267808c8db5a0b3a033e7361c5 Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Thu, 6 Mar 2025 14:34:59 +0500 Subject: [PATCH 1/4] =?UTF-8?q?=D0=9F=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=BD=D0=B0=20=D0=BD=D0=B0=D0=BB=D0=B8=D1=87=D0=B8?= =?UTF-8?q?=D0=B5=20HTML=20=D1=82=D1=8D=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index a94dbb1..98dfe9e 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "time" "github.com/go-yaml/yaml" @@ -47,6 +48,11 @@ func getDataFromConfig(path string) *mastodon.Config { return config } +func hasHTMLTags(s string) bool { + re := regexp.MustCompile(`<(?i)[a-z][a-z0-9]*[^>]*>`) + return re.MatchString(s) +} + func getKikiConfig(path string) KikiSettings { var kikiSettings KikiSettings kikiConfigFile, err := os.ReadFile(path) @@ -176,9 +182,11 @@ func main() { for range ticker.C { news := newsText(kikiConfig.RSSUri) if news[0].GUID != lastGUID { - createPost(news[0].Description) - lastGUID = news[0].GUID - log.Println("Пост отправлен") + if !hasHTMLTags(news[0].Description) { + createPost(news[0].Description) + lastGUID = news[0].GUID + log.Println("Пост отправлен") + } } /*for _, item := range newsText(kikiConfig.RSSUri) { if item.GUID == lastGUID { From c5caeecde88887bc9a02c85ccd2954126bad633d Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Thu, 10 Apr 2025 15:34:42 +0500 Subject: [PATCH 2/4] =?UTF-8?q?=D0=9E=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=B8=20=D0=BF=D0=BE=D1=81=D1=82=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3=20=D0=BA=D0=B0=D1=80=D1=82=D0=B8=D0=BD=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + config.example.yaml | 2 + go.mod | 18 ++++---- main.go | 100 ++++++++++++++++++++++++++++++++++++++------ 4 files changed, 101 insertions(+), 20 deletions(-) create mode 100644 config.example.yaml diff --git a/.gitignore b/.gitignore index 363e416..bdd4986 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ secret.conf go.sum +config.yaml \ No newline at end of file diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..3b02854 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,2 @@ +instance: https://pleroma.catgirls.asia +rss_url: https://4pda.to/feed \ No newline at end of file diff --git a/go.mod b/go.mod index 36ff9f2..4c8a8ee 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,27 @@ module kiki -go 1.21 +go 1.23.0 + +toolchain go1.24.2 require ( github.com/go-yaml/yaml v2.1.0+incompatible github.com/mattn/go-mastodon v0.0.9 github.com/mmcdole/gofeed v1.3.0 - github.com/urfave/cli/v3 v3.0.0-beta1 + github.com/urfave/cli/v3 v3.1.1 + golang.org/x/net v0.39.0 ) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/PuerkitoBio/goquery v1.10.2 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/mmcdole/goxpp v1.1.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/text v0.24.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/main.go b/main.go index 98dfe9e..07eab1b 100644 --- a/main.go +++ b/main.go @@ -3,17 +3,22 @@ package main import ( "context" "fmt" + "io" "log" + "net/http" "net/url" "os" "path/filepath" "regexp" + "strings" "time" "github.com/go-yaml/yaml" "github.com/mattn/go-mastodon" "github.com/mmcdole/gofeed" "github.com/urfave/cli/v3" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" ) type MastodonClientData struct { @@ -48,6 +53,25 @@ func getDataFromConfig(path string) *mastodon.Config { return config } +func returnImgArray(htmlText string) []string { + var imgArray []string + parser, err := html.Parse(strings.NewReader(htmlText)) + if err != nil { + log.Println(err) + } + for n := range parser.Descendants() { + if n.Type == html.ElementNode && n.DataAtom == atom.Img { + for _, img := range n.Attr { + if img.Key == "src" { + imgArray = append(imgArray, img.Val) + } + } + } + } + + return imgArray +} + func hasHTMLTags(s string) bool { re := regexp.MustCompile(`<(?i)[a-z][a-z0-9]*[^>]*>`) return re.MatchString(s) @@ -66,7 +90,7 @@ func getKikiConfig(path string) KikiSettings { return kikiSettings } -func ClientConfiguration(Instance string) { +func clientConfiguration(Instance string) { appConfig := &mastodon.AppConfig{ Server: Instance, ClientName: "Kiki", @@ -111,11 +135,13 @@ func ClientConfiguration(Instance string) { if err != nil { log.Println(err) } - secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE, 0o644) + log.Println(string(marshaledYaml)) + secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { log.Println(err) } - secretConfig.WriteString(string(marshaledYaml)) + secretConfig.Write(marshaledYaml) + defer secretConfig.Close() } func newsText(url string) []*gofeed.Item { @@ -144,6 +170,59 @@ func createPost(statusText string) { } } +func picBytesArray(picturesArray []string) [][]byte { + var picturesBytes [][]byte + for _, picture := range picturesArray { + resp, err := http.Get(picture) + if err != nil { + log.Println(err) + return picturesBytes + } + defer resp.Body.Close() + picBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Println(err) + return picturesBytes + } + picturesBytes = append(picturesBytes, picBytes) + } + return picturesBytes +} + +func uploadPictures(filesBytes [][]byte) []*mastodon.Attachment { + config := getDataFromConfig("secret.conf") + var attachments []*mastodon.Attachment + mastoClient := mastodon.NewClient(config) + for _, file := range filesBytes { + att, err := mastoClient.UploadMediaFromBytes(context.Background(), file) + if err != nil { + log.Println(err) + return attachments + } + attachments = append(attachments, att) + } + return attachments +} + +func postWithPictures(attachments []*mastodon.Attachment) { + var mediaID []mastodon.ID + config := getDataFromConfig("secret.conf") + mastoClient := mastodon.NewClient(config) + for _, attach := range attachments { + mediaID = append(mediaID, attach.ID) + } + toot := mastodon.Toot{ + MediaIDs: mediaID, + Visibility: "unlisted", + Sensitive: true, + } + _, err := mastoClient.PostStatus(context.Background(), &toot) + if err != nil { + log.Println(err) + } + +} + func main() { cmd := &cli.Command{ Name: "kiki", @@ -163,7 +242,7 @@ func main() { log.Println(err) } instanceUrlParser.Scheme = "https" - ClientConfiguration(instanceUrlParser.String()) + clientConfiguration(instanceUrlParser.String()) return nil }, }, @@ -183,19 +262,16 @@ func main() { news := newsText(kikiConfig.RSSUri) if news[0].GUID != lastGUID { if !hasHTMLTags(news[0].Description) { + log.Println(news[0].Description) createPost(news[0].Description) lastGUID = news[0].GUID log.Println("Пост отправлен") + } else { + attachments := uploadPictures(picBytesArray(returnImgArray(news[0].Description))) + postWithPictures(attachments) + lastGUID = news[0].GUID } } - /*for _, item := range newsText(kikiConfig.RSSUri) { - if item.GUID == lastGUID { - break - } - createPost(item.Description) - lastGUID = item.GUID - log.Println("Пост отправлен") - }*/ } return nil }, From d99d47e4e189305225272dc7865ae6eae9a49c8a Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Thu, 10 Apr 2025 19:29:33 +0500 Subject: [PATCH 3/4] =?UTF-8?q?=D0=98=D0=B7=D0=BC=D0=B5=D0=BD=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=BB=D0=BE=D0=B3=D0=B8=D0=BA=D1=83=20=D0=BE=D0=B1=D1=80?= =?UTF-8?q?=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B8=20=D1=82=D1=83=D1=82=D0=BE?= =?UTF-8?q?=D0=B2,=20=D1=82=D0=B5=D0=BF=D0=B5=D1=80=D1=8C=20=D0=B0=D0=B2?= =?UTF-8?q?=D1=82=D0=BE=D0=BC=D0=B0=D1=82=D0=BE=D0=BC=20=D0=BE=D1=82=D0=BF?= =?UTF-8?q?=D1=80=D0=B0=D0=B2=D1=8F=D0=B5=D1=82=D1=81=D1=8F=20=D0=B8=20?= =?UTF-8?q?=D1=82=D0=B5=D0=BA=D1=81=D1=82=20=D0=B2=20=D0=BF=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B5,=20=D0=B8=20=D0=BA=D0=B0=D1=80=D1=82=D0=B8=D0=BD?= =?UTF-8?q?=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 27 +++++++++++++++++++++++- config.yaml | 2 -- main.go | 59 +++++++++++++++-------------------------------------- 3 files changed, 43 insertions(+), 45 deletions(-) delete mode 100644 config.yaml diff --git a/.gitignore b/.gitignore index bdd4986..83d7ac2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,28 @@ secret.conf go.sum -config.yaml \ No newline at end of file +config.yaml + +### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# End of https://www.toptal.com/developers/gitignore/api/go diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 81b3139..0000000 --- a/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -instance: pleroma.catgirls.asia -rss_url: https://pleroma.catgirls.asia/users/SiberiaBread/feed.atom \ No newline at end of file diff --git a/main.go b/main.go index 07eab1b..41a3c2d 100644 --- a/main.go +++ b/main.go @@ -33,7 +33,7 @@ type KikiSettings struct { RSSUri string `yaml:"rss_url,omitempty"` } -func getDataFromConfig(path string) *mastodon.Config { +func getSecrets(path string) *mastodon.Config { var clientData MastodonClientData secretConfig, err := os.ReadFile(path) if err != nil { @@ -154,16 +154,7 @@ func newsText(url string) []*gofeed.Item { return feed.Items } -func createPost(statusText string) { - config := getDataFromConfig("secret.conf") - - mastoClient := mastodon.NewClient(config) - - toot := mastodon.Toot{ - Status: statusText, - Visibility: "unlisted", - } - +func createPost(mastoClient mastodon.Client, toot mastodon.Toot) { _, err := mastoClient.PostStatus(context.Background(), &toot) if err != nil { log.Println(err) @@ -189,10 +180,8 @@ func picBytesArray(picturesArray []string) [][]byte { return picturesBytes } -func uploadPictures(filesBytes [][]byte) []*mastodon.Attachment { - config := getDataFromConfig("secret.conf") +func uploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment { var attachments []*mastodon.Attachment - mastoClient := mastodon.NewClient(config) for _, file := range filesBytes { att, err := mastoClient.UploadMediaFromBytes(context.Background(), file) if err != nil { @@ -204,25 +193,6 @@ func uploadPictures(filesBytes [][]byte) []*mastodon.Attachment { return attachments } -func postWithPictures(attachments []*mastodon.Attachment) { - var mediaID []mastodon.ID - config := getDataFromConfig("secret.conf") - mastoClient := mastodon.NewClient(config) - for _, attach := range attachments { - mediaID = append(mediaID, attach.ID) - } - toot := mastodon.Toot{ - MediaIDs: mediaID, - Visibility: "unlisted", - Sensitive: true, - } - _, err := mastoClient.PostStatus(context.Background(), &toot) - if err != nil { - log.Println(err) - } - -} - func main() { cmd := &cli.Command{ Name: "kiki", @@ -256,21 +226,26 @@ func main() { log.Println(err) } kikiConfig := getKikiConfig(confFile) + mastoClient := mastodon.NewClient(getSecrets("secret.conf")) ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() for range ticker.C { news := newsText(kikiConfig.RSSUri) if news[0].GUID != lastGUID { - if !hasHTMLTags(news[0].Description) { - log.Println(news[0].Description) - createPost(news[0].Description) - lastGUID = news[0].GUID - log.Println("Пост отправлен") - } else { - attachments := uploadPictures(picBytesArray(returnImgArray(news[0].Description))) - postWithPictures(attachments) - lastGUID = news[0].GUID + toot := mastodon.Toot{ + Visibility: "unlisted", + Sensitive: true, } + log.Println(news[0].Description) + toot.Status = news[0].Description + lastGUID = news[0].GUID + if hasHTMLTags(news[0].Description) { + attachments := uploadPictures(*mastoClient, picBytesArray(returnImgArray(news[0].Description))) + for _, attach := range attachments { + toot.MediaIDs = append(toot.MediaIDs, attach.ID) + } + } + createPost(*mastoClient, toot) } } return nil From 4d1d1360dc3263151865d1ae2a31f63fbd20fc40 Mon Sep 17 00:00:00 2001 From: B4D_US3R Date: Fri, 11 Apr 2025 17:13:54 +0500 Subject: [PATCH 4/4] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=BE=D0=B9=20=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80?= =?UTF-8?q?=D0=B8=D0=BD=D0=B3,=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BF=D1=80=D0=B8=D0=BD=D1=86=D0=B8=D0=BF?= =?UTF-8?q?=D0=B0=20=D1=81=D0=BE=D0=B7=D0=B4=D0=B0=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D1=81=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 106 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 39 deletions(-) diff --git a/main.go b/main.go index 41a3c2d..6b00be9 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( "net/url" "os" "path/filepath" - "regexp" "strings" "time" @@ -18,7 +17,6 @@ import ( "github.com/mmcdole/gofeed" "github.com/urfave/cli/v3" "golang.org/x/net/html" - "golang.org/x/net/html/atom" ) type MastodonClientData struct { @@ -35,14 +33,17 @@ type KikiSettings struct { func getSecrets(path string) *mastodon.Config { var clientData MastodonClientData + secretConfig, err := os.ReadFile(path) if err != nil { log.Println(err) } + err = yaml.Unmarshal(secretConfig, &clientData) if err != nil { log.Println(err) } + config := &mastodon.Config{ Server: clientData.Instance, ClientID: clientData.ClientID, @@ -53,40 +54,19 @@ func getSecrets(path string) *mastodon.Config { return config } -func returnImgArray(htmlText string) []string { - var imgArray []string - parser, err := html.Parse(strings.NewReader(htmlText)) - if err != nil { - log.Println(err) - } - for n := range parser.Descendants() { - if n.Type == html.ElementNode && n.DataAtom == atom.Img { - for _, img := range n.Attr { - if img.Key == "src" { - imgArray = append(imgArray, img.Val) - } - } - } - } - - return imgArray -} - -func hasHTMLTags(s string) bool { - re := regexp.MustCompile(`<(?i)[a-z][a-z0-9]*[^>]*>`) - return re.MatchString(s) -} - func getKikiConfig(path string) KikiSettings { var kikiSettings KikiSettings + kikiConfigFile, err := os.ReadFile(path) if err != nil { log.Println(err) } + err = yaml.Unmarshal(kikiConfigFile, &kikiSettings) if err != nil { log.Println(err) } + return kikiSettings } @@ -98,6 +78,7 @@ func clientConfiguration(Instance string) { Website: "catgirls.asia", RedirectURIs: "urn:ietf:wg:oauth:2.0:oob", } + app, err := mastodon.RegisterApp(context.Background(), appConfig) if err != nil { log.Println(err) @@ -135,6 +116,7 @@ func clientConfiguration(Instance string) { if err != nil { log.Println(err) } + log.Println(string(marshaledYaml)) secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { @@ -182,17 +164,62 @@ func picBytesArray(picturesArray []string) [][]byte { func uploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment { var attachments []*mastodon.Attachment + for _, file := range filesBytes { att, err := mastoClient.UploadMediaFromBytes(context.Background(), file) + if err != nil { log.Println(err) return attachments } + attachments = append(attachments, att) } + return attachments } +func createToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) { + var tootText string + var imgArray []string + var attachments []*mastodon.Attachment + + toot := mastodon.Toot{ + Visibility: "unlisted", + Sensitive: true, + } + + uString := html.UnescapeString(newsDesc) + pHtml, err := html.Parse(strings.NewReader(uString)) + if err != nil { + return mastodon.Toot{}, err + } + + for n := range pHtml.Descendants() { + if n.Type != html.ElementNode { + tootText += (n.Data + "\n") + } + if n.Type == html.ElementNode && n.Data == "img" { + for _, attr := range n.Attr { + if attr.Key == "src" { + imgArray = append(imgArray, attr.Val) + } + } + } + } + + if len(imgArray) != 0 { + attachments = uploadPictures(mastoClient, picBytesArray(imgArray)) + } + + toot.Status = tootText + for _, attach := range attachments { + toot.MediaIDs = append(toot.MediaIDs, attach.ID) + } + + return toot, nil +} + func main() { cmd := &cli.Command{ Name: "kiki", @@ -203,16 +230,20 @@ func main() { Usage: "Инициализировать клиента", Action: func(ctx context.Context, cmd *cli.Command) error { confFile, err := filepath.Abs("config.yaml") + kikiConfig := getKikiConfig(confFile) + if err != nil { log.Println(err) } - kikiConfig := getKikiConfig(confFile) + instanceUrlParser, err := url.Parse(kikiConfig.Instance) if err != nil { log.Println(err) } instanceUrlParser.Scheme = "https" + clientConfiguration(instanceUrlParser.String()) + return nil }, }, @@ -221,31 +252,28 @@ func main() { Usage: "Запуск транслятора", Action: func(ctx context.Context, cmd *cli.Command) error { var lastGUID string + mastoClient := mastodon.NewClient(getSecrets("secret.conf")) + confFile, err := filepath.Abs("config.yaml") if err != nil { log.Println(err) } kikiConfig := getKikiConfig(confFile) - mastoClient := mastodon.NewClient(getSecrets("secret.conf")) + ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() for range ticker.C { news := newsText(kikiConfig.RSSUri) if news[0].GUID != lastGUID { - toot := mastodon.Toot{ - Visibility: "unlisted", - Sensitive: true, - } log.Println(news[0].Description) - toot.Status = news[0].Description - lastGUID = news[0].GUID - if hasHTMLTags(news[0].Description) { - attachments := uploadPictures(*mastoClient, picBytesArray(returnImgArray(news[0].Description))) - for _, attach := range attachments { - toot.MediaIDs = append(toot.MediaIDs, attach.ID) - } + + toot, err := createToot(*mastoClient, news[0].Description) + if err != nil { + log.Println(err) } + createPost(*mastoClient, toot) + lastGUID = news[0].GUID } } return nil