Merge pull request 'img_extractor' (#1) from img_extractor into master

Reviewed-on: #1

.gitignore

@@ -1,2 +1,28 @@
 secret.conf
 go.sum
+config.yaml
+
+### Go ###
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+
+# End of https://www.toptal.com/developers/gitignore/api/go

config.example.yaml

@@ -0,0 +1,2 @@
+instance: https://pleroma.catgirls.asia
+rss_url: https://4pda.to/feed

config.yaml

@@ -1,2 +0,0 @@
-instance: pleroma.catgirls.asia
-rss_url: https://pleroma.catgirls.asia/users/SiberiaBread/feed.atom

go.mod

@@ -1,25 +1,27 @@
 module kiki
 
-go 1.21
+go 1.23.0
+
+toolchain go1.24.2
 
 require (
 	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/mattn/go-mastodon v0.0.9
 	github.com/mmcdole/gofeed v1.3.0
-	github.com/urfave/cli/v3 v3.0.0-beta1
+	github.com/urfave/cli/v3 v3.1.1
+	golang.org/x/net v0.39.0
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/PuerkitoBio/goquery v1.10.2 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
-	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/mmcdole/goxpp v1.1.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
-	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

main.go

@@ -3,16 +3,20 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/go-yaml/yaml"
 	"github.com/mattn/go-mastodon"
 	"github.com/mmcdole/gofeed"
 	"github.com/urfave/cli/v3"
+	"golang.org/x/net/html"
 )
 
 type MastodonClientData struct {
@@ -27,16 +31,19 @@ type KikiSettings struct {
 	RSSUri   string `yaml:"rss_url,omitempty"`
 }
 
-func getDataFromConfig(path string) *mastodon.Config {
+func getSecrets(path string) *mastodon.Config {
 	var clientData MastodonClientData
+
 	secretConfig, err := os.ReadFile(path)
 	if err != nil {
 		log.Println(err)
 	}
+
 	err = yaml.Unmarshal(secretConfig, &clientData)
 	if err != nil {
 		log.Println(err)
 	}
+
 	config := &mastodon.Config{
 		Server:       clientData.Instance,
 		ClientID:     clientData.ClientID,
@@ -49,18 +56,21 @@ func getDataFromConfig(path string) *mastodon.Config {
 
 func getKikiConfig(path string) KikiSettings {
 	var kikiSettings KikiSettings
+
 	kikiConfigFile, err := os.ReadFile(path)
 	if err != nil {
 		log.Println(err)
 	}
+
 	err = yaml.Unmarshal(kikiConfigFile, &kikiSettings)
 	if err != nil {
 		log.Println(err)
 	}
+
 	return kikiSettings
 }
 
-func ClientConfiguration(Instance string) {
+func clientConfiguration(Instance string) {
 	appConfig := &mastodon.AppConfig{
 		Server:       Instance,
 		ClientName:   "Kiki",
@@ -68,6 +78,7 @@ func ClientConfiguration(Instance string) {
 		Website:      "catgirls.asia",
 		RedirectURIs: "urn:ietf:wg:oauth:2.0:oob",
 	}
+
 	app, err := mastodon.RegisterApp(context.Background(), appConfig)
 	if err != nil {
 		log.Println(err)
@@ -105,11 +116,14 @@ func ClientConfiguration(Instance string) {
 	if err != nil {
 		log.Println(err)
 	}
-	secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE, 0o644)
+
+	log.Println(string(marshaledYaml))
+	secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
 	if err != nil {
 		log.Println(err)
 	}
-	secretConfig.WriteString(string(marshaledYaml))
+	secretConfig.Write(marshaledYaml)
+	defer secretConfig.Close()
 }
 
 func newsText(url string) []*gofeed.Item {
@@ -122,22 +136,90 @@ func newsText(url string) []*gofeed.Item {
 	return feed.Items
 }
 
-func createPost(statusText string) {
+func createPost(mastoClient mastodon.Client, toot mastodon.Toot) {
-	config := getDataFromConfig("secret.conf")
-
-	mastoClient := mastodon.NewClient(config)
-
-	toot := mastodon.Toot{
-		Status:     statusText,
-		Visibility: "unlisted",
-	}
-
 	_, err := mastoClient.PostStatus(context.Background(), &toot)
 	if err != nil {
 		log.Println(err)
 	}
 }
 
+func picBytesArray(picturesArray []string) [][]byte {
+	var picturesBytes [][]byte
+	for _, picture := range picturesArray {
+		resp, err := http.Get(picture)
+		if err != nil {
+			log.Println(err)
+			return picturesBytes
+		}
+		defer resp.Body.Close()
+		picBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			log.Println(err)
+			return picturesBytes
+		}
+		picturesBytes = append(picturesBytes, picBytes)
+	}
+	return picturesBytes
+}
+
+func uploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment {
+	var attachments []*mastodon.Attachment
+
+	for _, file := range filesBytes {
+		att, err := mastoClient.UploadMediaFromBytes(context.Background(), file)
+
+		if err != nil {
+			log.Println(err)
+			return attachments
+		}
+
+		attachments = append(attachments, att)
+	}
+
+	return attachments
+}
+
+func createToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) {
+	var tootText string
+	var imgArray []string
+	var attachments []*mastodon.Attachment
+
+	toot := mastodon.Toot{
+		Visibility: "unlisted",
+		Sensitive:  true,
+	}
+
+	uString := html.UnescapeString(newsDesc)
+	pHtml, err := html.Parse(strings.NewReader(uString))
+	if err != nil {
+		return mastodon.Toot{}, err
+	}
+
+	for n := range pHtml.Descendants() {
+		if n.Type != html.ElementNode {
+			tootText += (n.Data + "\n")
+		}
+		if n.Type == html.ElementNode && n.Data == "img" {
+			for _, attr := range n.Attr {
+				if attr.Key == "src" {
+					imgArray = append(imgArray, attr.Val)
+				}
+			}
+		}
+	}
+
+	if len(imgArray) != 0 {
+		attachments = uploadPictures(mastoClient, picBytesArray(imgArray))
+	}
+
+	toot.Status = tootText
+	for _, attach := range attachments {
+		toot.MediaIDs = append(toot.MediaIDs, attach.ID)
+	}
+
+	return toot, nil
+}
+
 func main() {
 	cmd := &cli.Command{
 		Name:  "kiki",
@@ -148,16 +230,20 @@ func main() {
 				Usage: "Инициализировать клиента",
 				Action: func(ctx context.Context, cmd *cli.Command) error {
 					confFile, err := filepath.Abs("config.yaml")
+					kikiConfig := getKikiConfig(confFile)
+
 					if err != nil {
 						log.Println(err)
 					}
-					kikiConfig := getKikiConfig(confFile)
+
 					instanceUrlParser, err := url.Parse(kikiConfig.Instance)
 					if err != nil {
 						log.Println(err)
 					}
 					instanceUrlParser.Scheme = "https"
-					ClientConfiguration(instanceUrlParser.String())
+
+					clientConfiguration(instanceUrlParser.String())
+
 					return nil
 				},
 			},
@@ -166,28 +252,29 @@ func main() {
 				Usage: "Запуск транслятора",
 				Action: func(ctx context.Context, cmd *cli.Command) error {
 					var lastGUID string
+					mastoClient := mastodon.NewClient(getSecrets("secret.conf"))
+
 					confFile, err := filepath.Abs("config.yaml")
 					if err != nil {
 						log.Println(err)
 					}
 					kikiConfig := getKikiConfig(confFile)
+
 					ticker := time.NewTicker(1 * time.Minute)
 					defer ticker.Stop()
 					for range ticker.C {
 						news := newsText(kikiConfig.RSSUri)
 						if news[0].GUID != lastGUID {
-							createPost(news[0].Description)
+							log.Println(news[0].Description)
+
+							toot, err := createToot(*mastoClient, news[0].Description)
+							if err != nil {
+								log.Println(err)
+							}
+
+							createPost(*mastoClient, toot)
 							lastGUID = news[0].GUID
-							log.Println("Пост отправлен")
 						}
-						/*for _, item := range newsText(kikiConfig.RSSUri) {
-							if item.GUID == lastGUID {
-								break
-							}
-							createPost(item.Description)
-							lastGUID = item.GUID
-							log.Println("Пост отправлен")
-						}*/
 					}
 					return nil
 				},