img_extractor #1

Merged
B4D_US3R merged 4 commits from img_extractor into master 2025-04-23 12:50:25 +00:00
4 changed files with 101 additions and 20 deletions
Showing only changes of commit c5caeecde8 - Show all commits

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
secret.conf secret.conf
go.sum go.sum
config.yaml

2
config.example.yaml Normal file
View file

@ -0,0 +1,2 @@
instance: https://pleroma.catgirls.asia
rss_url: https://4pda.to/feed

18
go.mod
View file

@ -1,25 +1,27 @@
module kiki module kiki
go 1.21 go 1.23.0
toolchain go1.24.2
require ( require (
github.com/go-yaml/yaml v2.1.0+incompatible github.com/go-yaml/yaml v2.1.0+incompatible
github.com/mattn/go-mastodon v0.0.9 github.com/mattn/go-mastodon v0.0.9
github.com/mmcdole/gofeed v1.3.0 github.com/mmcdole/gofeed v1.3.0
github.com/urfave/cli/v3 v3.0.0-beta1 github.com/urfave/cli/v3 v3.1.1
golang.org/x/net v0.39.0
) )
require ( require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/PuerkitoBio/goquery v1.10.2 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/gorilla/websocket v1.5.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect github.com/mmcdole/goxpp v1.1.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
golang.org/x/net v0.25.0 // indirect golang.org/x/text v0.24.0 // indirect
golang.org/x/text v0.15.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
) )

100
main.go
View file

@ -3,17 +3,22 @@ package main
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"log" "log"
"net/http"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strings"
"time" "time"
"github.com/go-yaml/yaml" "github.com/go-yaml/yaml"
"github.com/mattn/go-mastodon" "github.com/mattn/go-mastodon"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
) )
type MastodonClientData struct { type MastodonClientData struct {
@ -48,6 +53,25 @@ func getDataFromConfig(path string) *mastodon.Config {
return config return config
} }
func returnImgArray(htmlText string) []string {
var imgArray []string
parser, err := html.Parse(strings.NewReader(htmlText))
if err != nil {
log.Println(err)
}
for n := range parser.Descendants() {
if n.Type == html.ElementNode && n.DataAtom == atom.Img {
for _, img := range n.Attr {
if img.Key == "src" {
imgArray = append(imgArray, img.Val)
}
}
}
}
return imgArray
}
func hasHTMLTags(s string) bool { func hasHTMLTags(s string) bool {
re := regexp.MustCompile(`<(?i)[a-z][a-z0-9]*[^>]*>`) re := regexp.MustCompile(`<(?i)[a-z][a-z0-9]*[^>]*>`)
return re.MatchString(s) return re.MatchString(s)
@ -66,7 +90,7 @@ func getKikiConfig(path string) KikiSettings {
return kikiSettings return kikiSettings
} }
func ClientConfiguration(Instance string) { func clientConfiguration(Instance string) {
appConfig := &mastodon.AppConfig{ appConfig := &mastodon.AppConfig{
Server: Instance, Server: Instance,
ClientName: "Kiki", ClientName: "Kiki",
@ -111,11 +135,13 @@ func ClientConfiguration(Instance string) {
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE, 0o644) log.Println(string(marshaledYaml))
secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
secretConfig.WriteString(string(marshaledYaml)) secretConfig.Write(marshaledYaml)
defer secretConfig.Close()
} }
func newsText(url string) []*gofeed.Item { func newsText(url string) []*gofeed.Item {
@ -144,6 +170,59 @@ func createPost(statusText string) {
} }
} }
func picBytesArray(picturesArray []string) [][]byte {
var picturesBytes [][]byte
for _, picture := range picturesArray {
resp, err := http.Get(picture)
if err != nil {
log.Println(err)
return picturesBytes
}
defer resp.Body.Close()
picBytes, err := io.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return picturesBytes
}
picturesBytes = append(picturesBytes, picBytes)
}
return picturesBytes
}
func uploadPictures(filesBytes [][]byte) []*mastodon.Attachment {
config := getDataFromConfig("secret.conf")
var attachments []*mastodon.Attachment
mastoClient := mastodon.NewClient(config)
for _, file := range filesBytes {
att, err := mastoClient.UploadMediaFromBytes(context.Background(), file)
if err != nil {
log.Println(err)
return attachments
}
attachments = append(attachments, att)
}
return attachments
}
func postWithPictures(attachments []*mastodon.Attachment) {
var mediaID []mastodon.ID
config := getDataFromConfig("secret.conf")
mastoClient := mastodon.NewClient(config)
for _, attach := range attachments {
mediaID = append(mediaID, attach.ID)
}
toot := mastodon.Toot{
MediaIDs: mediaID,
Visibility: "unlisted",
Sensitive: true,
}
_, err := mastoClient.PostStatus(context.Background(), &toot)
if err != nil {
log.Println(err)
}
}
func main() { func main() {
cmd := &cli.Command{ cmd := &cli.Command{
Name: "kiki", Name: "kiki",
@ -163,7 +242,7 @@ func main() {
log.Println(err) log.Println(err)
} }
instanceUrlParser.Scheme = "https" instanceUrlParser.Scheme = "https"
ClientConfiguration(instanceUrlParser.String()) clientConfiguration(instanceUrlParser.String())
return nil return nil
}, },
}, },
@ -183,19 +262,16 @@ func main() {
news := newsText(kikiConfig.RSSUri) news := newsText(kikiConfig.RSSUri)
if news[0].GUID != lastGUID { if news[0].GUID != lastGUID {
if !hasHTMLTags(news[0].Description) { if !hasHTMLTags(news[0].Description) {
log.Println(news[0].Description)
createPost(news[0].Description) createPost(news[0].Description)
lastGUID = news[0].GUID lastGUID = news[0].GUID
log.Println("Пост отправлен") log.Println("Пост отправлен")
} else {
attachments := uploadPictures(picBytesArray(returnImgArray(news[0].Description)))
postWithPictures(attachments)
lastGUID = news[0].GUID
} }
} }
/*for _, item := range newsText(kikiConfig.RSSUri) {
if item.GUID == lastGUID {
break
}
createPost(item.Description)
lastGUID = item.GUID
log.Println("Пост отправлен")
}*/
} }
return nil return nil
}, },