B4D_US3R/kiki
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add_redis #2

Merged
B4D_US3R merged 7 commits from add_redis into master 2025-04-23 17:51:51 +05:00
Conversation 0 Commits 7 Files changed 13 +484 -301
13 changed files with 484 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -18,7 +18,7 @@ config.yaml
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
kiki
# Dependency directories (remove the comment below to include it)
# vendor/

16
Dockerfile Normal file
View file

@ -0,0 +1,16 @@
FROM golang:1.24
WORKDIR /app/kiki
RUN mkdir /app/kiki/config /app/kiki/service /app/kiki/stacker /app/kiki/tooter
COPY config/* /app/kiki/config
COPY service/* /app/kiki/service
COPY stacker/* /app/kiki/stacker
COPY tooter/* /app/kiki/tooter
RUN cd service && go mod tidy && cd ../
RUN go build -C ./service -o ../kiki
CMD [ "/app/kiki/kiki", "run" ]

5
config.example.yaml
View file

@ -1,2 +1,5 @@
instance: https://pleroma.catgirls.asia
rss_url: https://4pda.to/feed
rss_url: https://4pda.to/feed
sensitive: true
redis:
address: localhost:6379

64
config/config.go Normal file
View file

@ -0,0 +1,64 @@
package config
import (
"log"
"os"
"github.com/mattn/go-mastodon"
"gopkg.in/yaml.v2"
)
type KikiSettings struct {
Instance string `yaml:"instance,omitempty"`
RSSUri string `yaml:"rss_url,omitempty"`
Sensitive bool `yaml:"sensitive,omitempty"`
Redis struct {
Address string `yaml:"address"`
} `yaml:"redis"`
}
type MastodonClientData struct {
ClientID string `yaml:"clientID,omitempty"`
ClientSecret string `yaml:"clientSecret,omitempty"`
AccessToken string `yaml:"accessToken,omitempty"`
Instance string `yaml:"instance,omitempty"`
}
func GetKikiConfig(path string) KikiSettings {
var kikiSettings KikiSettings
kikiConfigFile, err := os.ReadFile(path)
if err != nil {
log.Println(err)
}
err = yaml.Unmarshal(kikiConfigFile, &kikiSettings)
if err != nil {
log.Println(err)
}
return kikiSettings
}
func GetSecrets(path string) *mastodon.Config {
var clientData MastodonClientData
secretConfig, err := os.ReadFile(path)
if err != nil {
log.Println(err)
}
err = yaml.Unmarshal(secretConfig, &clientData)
if err != nil {
log.Println(err)
}
config := &mastodon.Config{
Server: clientData.Instance,
ClientID: clientData.ClientID,
ClientSecret: clientData.ClientSecret,
AccessToken: clientData.AccessToken,
}
return config
}

14
config/go.mod Normal file
View file

@ -0,0 +1,14 @@
module kiki/config
go 1.24.2
require (
github.com/mattn/go-mastodon v0.0.9
gopkg.in/yaml.v2 v2.4.0
)
require (
github.com/gorilla/websocket v1.5.1 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
golang.org/x/net v0.25.0 // indirect
)

13
docker-compose.yml Normal file
View file

@ -0,0 +1,13 @@
services:
kiki:
build: ./
restart: always
volumes:
- ./config.yaml:/app/kiki/config.yaml
- ./secret.conf:/app/kiki/secret.conf
depends_on:
- redis
redis:
image: redis
restart: always

287
main.go
View file

@ -1,287 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/go-yaml/yaml"
"github.com/mattn/go-mastodon"
"github.com/mmcdole/gofeed"
"github.com/urfave/cli/v3"
"golang.org/x/net/html"
)
type MastodonClientData struct {
ClientID string `yaml:"clientID,omitempty"`
ClientSecret string `yaml:"clientSecret,omitempty"`
AccessToken string `yaml:"accessToken,omitempty"`
Instance string `yaml:"instance,omitempty"`
}
type KikiSettings struct {
Instance string `yaml:"instance,omitempty"`
RSSUri string `yaml:"rss_url,omitempty"`
}
func getSecrets(path string) *mastodon.Config {
var clientData MastodonClientData
secretConfig, err := os.ReadFile(path)
if err != nil {
log.Println(err)
}
err = yaml.Unmarshal(secretConfig, &clientData)
if err != nil {
log.Println(err)
}
config := &mastodon.Config{
Server: clientData.Instance,
ClientID: clientData.ClientID,
ClientSecret: clientData.ClientSecret,
AccessToken: clientData.AccessToken,
}
return config
}
func getKikiConfig(path string) KikiSettings {
var kikiSettings KikiSettings
kikiConfigFile, err := os.ReadFile(path)
if err != nil {
log.Println(err)
}
err = yaml.Unmarshal(kikiConfigFile, &kikiSettings)
if err != nil {
log.Println(err)
}
return kikiSettings
}
func clientConfiguration(Instance string) {
appConfig := &mastodon.AppConfig{
Server: Instance,
ClientName: "Kiki",
Scopes: "read write follow",
Website: "catgirls.asia",
RedirectURIs: "urn:ietf:wg:oauth:2.0:oob",
}
app, err := mastodon.RegisterApp(context.Background(), appConfig)
if err != nil {
log.Println(err)
}
u, err := url.Parse(app.AuthURI)
if err != nil {
log.Println(err)
}
var userToken string
fmt.Println(u)
fmt.Scanln(&userToken)
config := &mastodon.Config{
Server: Instance,
ClientID: app.ClientID,
ClientSecret: app.ClientSecret,
AccessToken: userToken,
}
mastoClient := mastodon.NewClient(config)
err = mastoClient.AuthenticateToken(context.Background(), userToken, "urn:ietf:wg:oauth:2.0:oob")
if err != nil {
log.Println(err)
}
clientData := MastodonClientData{
Instance: Instance,
ClientID: mastoClient.Config.ClientID,
ClientSecret: mastoClient.Config.ClientSecret,
AccessToken: mastoClient.Config.AccessToken,
}
marshaledYaml, err := yaml.Marshal(clientData)
if err != nil {
log.Println(err)
}
log.Println(string(marshaledYaml))
secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
log.Println(err)
}
secretConfig.Write(marshaledYaml)
defer secretConfig.Close()
}
func newsText(url string) []*gofeed.Item {
fp := gofeed.NewParser()
feed, err := fp.ParseURL(url)
if err != nil {
log.Println(err)
}
log.Println("RSS лента получена")
return feed.Items
}
func createPost(mastoClient mastodon.Client, toot mastodon.Toot) {
_, err := mastoClient.PostStatus(context.Background(), &toot)
if err != nil {
log.Println(err)
}
}
func picBytesArray(picturesArray []string) [][]byte {
var picturesBytes [][]byte
for _, picture := range picturesArray {
resp, err := http.Get(picture)
if err != nil {
log.Println(err)
return picturesBytes
}
defer resp.Body.Close()
picBytes, err := io.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return picturesBytes
}
picturesBytes = append(picturesBytes, picBytes)
}
return picturesBytes
}
func uploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment {
var attachments []*mastodon.Attachment
for _, file := range filesBytes {
att, err := mastoClient.UploadMediaFromBytes(context.Background(), file)
if err != nil {
log.Println(err)
return attachments
}
attachments = append(attachments, att)
}
return attachments
}
func createToot(mastoClient mastodon.Client, newsDesc string) (mastodon.Toot, error) {
var tootText string
var imgArray []string
var attachments []*mastodon.Attachment
toot := mastodon.Toot{
Visibility: "unlisted",
Sensitive: true,
}
uString := html.UnescapeString(newsDesc)
pHtml, err := html.Parse(strings.NewReader(uString))
if err != nil {
return mastodon.Toot{}, err
}
for n := range pHtml.Descendants() {
if n.Type != html.ElementNode {
tootText += (n.Data + "\n")
}
if n.Type == html.ElementNode && n.Data == "img" {
for _, attr := range n.Attr {
if attr.Key == "src" {
imgArray = append(imgArray, attr.Val)
}
}
}
}
if len(imgArray) != 0 {
attachments = uploadPictures(mastoClient, picBytesArray(imgArray))
}
toot.Status = tootText
for _, attach := range attachments {
toot.MediaIDs = append(toot.MediaIDs, attach.ID)
}
return toot, nil
}
func main() {
cmd := &cli.Command{
Name: "kiki",
Usage: "Ретранслятор из RSS в Mastodon. Когда-нибудь...",
Commands: []*cli.Command{
{
Name: "init",
Usage: "Инициализировать клиента",
Action: func(ctx context.Context, cmd *cli.Command) error {
confFile, err := filepath.Abs("config.yaml")
kikiConfig := getKikiConfig(confFile)
if err != nil {
log.Println(err)
}
instanceUrlParser, err := url.Parse(kikiConfig.Instance)
if err != nil {
log.Println(err)
}
instanceUrlParser.Scheme = "https"
clientConfiguration(instanceUrlParser.String())
return nil
},
},
{
Name: "run",
Usage: "Запуск транслятора",
Action: func(ctx context.Context, cmd *cli.Command) error {
var lastGUID string
mastoClient := mastodon.NewClient(getSecrets("secret.conf"))
confFile, err := filepath.Abs("config.yaml")
if err != nil {
log.Println(err)
}
kikiConfig := getKikiConfig(confFile)
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for range ticker.C {
news := newsText(kikiConfig.RSSUri)
if news[0].GUID != lastGUID {
log.Println(news[0].Description)
toot, err := createToot(*mastoClient, news[0].Description)
if err != nil {
log.Println(err)
}
createPost(*mastoClient, toot)
lastGUID = news[0].GUID
}
}
return nil
},
},
},
}
if err := cmd.Run(context.Background(), os.Args); err != nil {
log.Fatal(err)
}
}

35
service/go.mod Normal file
View file

@ -0,0 +1,35 @@
module kiki/kiki
go 1.24.2
replace kiki/config => ../config
replace kiki/stacker => ../stacker
replace kiki/tooter => ../tooter
require (
github.com/mattn/go-mastodon v0.0.9
github.com/urfave/cli/v3 v3.2.0
kiki/config v0.0.0-00010101000000-000000000000
kiki/stacker v0.0.0-00010101000000-000000000000
kiki/tooter v0.0.0-00010101000000-000000000000
)
require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mmcdole/gofeed v1.3.0 // indirect
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/redis/go-redis/v9 v9.7.3 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
golang.org/x/net v0.39.0 // indirect
golang.org/x/text v0.24.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

95
service/kiki.go Normal file
View file

@ -0,0 +1,95 @@
package main
import (
"context"
"log"
"net/url"
"os"
"path/filepath"
"time"
"kiki/config"
"kiki/stacker"
"kiki/tooter"
"github.com/mattn/go-mastodon"
"github.com/urfave/cli/v3"
)
func main() {
cmd := &cli.Command{
Name: "kiki",
Usage: "Ретранслятор из RSS в Mastodon. Когда-нибудь...",
Commands: []*cli.Command{
{
Name: "init",
Usage: "Инициализировать клиента",
Action: func(ctx context.Context, cmd *cli.Command) error {
confFile, err := filepath.Abs("config.yaml")
kikiConfig := config.GetKikiConfig(confFile)
if err != nil {
log.Println(err)
}
instanceUrlParser, err := url.Parse(kikiConfig.Instance)
if err != nil {
log.Println(err)
}
instanceUrlParser.Scheme = "https"
tooter.ClientConfiguration(instanceUrlParser.String())
return nil
},
},
{
Name: "run",
Usage: "Запуск транслятора",
Action: func(ctx context.Context, cmd *cli.Command) error {
mastoClient := mastodon.NewClient(config.GetSecrets("secret.conf"))
confFile, err := filepath.Abs("config.yaml")
if err != nil {
log.Println(err)
}
kikiConfig := config.GetKikiConfig(confFile)
rdb := stacker.ConnectToRedis(kikiConfig.Redis.Address)
defer stacker.SaveRedis(rdb)
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for range ticker.C {
newPosts := tooter.NewsText(kikiConfig.RSSUri)
for _, post := range newPosts {
inStack, err := stacker.CheckInRedis(rdb, post.GUID)
if err != nil {
log.Println(err)
}
if !inStack {
log.Println(post.Description)
toot, err := tooter.CreateToot(*mastoClient, post, kikiConfig.Sensitive)
if err != nil {
log.Println(err)
}
tooter.CreatePost(*mastoClient, toot)
stacker.SetToRedis(rdb, post.GUID, post.Description)
}
}
}
return nil
},
},
},
}
if err := cmd.Run(context.Background(), os.Args); err != nil {
log.Fatal(err)
}
}

10
stacker/go.mod Normal file
View file

@ -0,0 +1,10 @@
module kiki/stacker
go 1.24.2
require github.com/redis/go-redis/v9 v9.7.3
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
)

43
stacker/stacker.go Normal file
View file

@ -0,0 +1,43 @@
package stacker
import (
"context"
"github.com/redis/go-redis/v9"
)
func ConnectToRedis(addr string) redis.Client {
rdb := redis.NewClient(&redis.Options{
Addr: addr,
Password: "",
DB: 0,
})
return *rdb
}
func SetToRedis(rdb redis.Client, key string, val interface{}) error {
err := rdb.Set(context.Background(), key, val, 0).Err()
if err != nil {
return err
}
return nil
}
func CheckInRedis(rdb redis.Client, key string) (bool, error) {
_, err := rdb.Get(context.Background(), key).Result()
if err == redis.Nil {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
func SaveRedis(rdb redis.Client) error {
err := rdb.Save(context.Background()).Err()
if err != nil {
return err
}
return nil
}

22
go.mod → tooter/go.mod
View file

@ -1,27 +1,25 @@
module kiki
module kiki/tooter
go 1.23.0
toolchain go1.24.2
go 1.24.2
require (
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/mattn/go-mastodon v0.0.9
github.com/mmcdole/gofeed v1.3.0
github.com/urfave/cli/v3 v3.1.1
golang.org/x/net v0.39.0
gopkg.in/yaml.v2 v2.4.0
kiki/config v0.0.0-00010101000000-000000000000
)
require (
github.com/PuerkitoBio/goquery v1.10.2 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mmcdole/goxpp v1.1.1 // indirect
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
golang.org/x/text v0.24.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
replace kiki/config => ../config

179
tooter/tooter.go Normal file
View file

@ -0,0 +1,179 @@
package tooter
import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"strings"
"kiki/config"
"github.com/mattn/go-mastodon"
"github.com/mmcdole/gofeed"
"golang.org/x/net/html"
"gopkg.in/yaml.v2"
)
func ClientConfiguration(Instance string) {
appConfig := &mastodon.AppConfig{
Server: Instance,
ClientName: "Kiki",
Scopes: "read write follow",
Website: "catgirls.asia",
RedirectURIs: "urn:ietf:wg:oauth:2.0:oob",
}
app, err := mastodon.RegisterApp(context.Background(), appConfig)
if err != nil {
log.Println(err)
}
u, err := url.Parse(app.AuthURI)
if err != nil {
log.Println(err)
}
var userToken string
//fmt.Println(u)
fmt.Printf("Перейдите по ссылке\n%s\nИ введите user token ниже:\n", u)
fmt.Scanln(&userToken)
conf := &mastodon.Config{
Server: Instance,
ClientID: app.ClientID,
ClientSecret: app.ClientSecret,
AccessToken: userToken,
}
mastoClient := mastodon.NewClient(conf)
err = mastoClient.AuthenticateToken(context.Background(), userToken, "urn:ietf:wg:oauth:2.0:oob")
if err != nil {
log.Println(err)
}
clientData := config.MastodonClientData{
Instance: Instance,
ClientID: mastoClient.Config.ClientID,
ClientSecret: mastoClient.Config.ClientSecret,
AccessToken: mastoClient.Config.AccessToken,
}
marshaledYaml, err := yaml.Marshal(clientData)
if err != nil {
log.Println(err)
}
log.Println(string(marshaledYaml))
secretConfig, err := os.OpenFile("secret.conf", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
log.Println(err)
}
secretConfig.Write(marshaledYaml)
defer secretConfig.Close()
}
func NewsText(url string) []*gofeed.Item {
fp := gofeed.NewParser()
feed, err := fp.ParseURL(url)
if err != nil {
log.Println(err)
}
log.Println("RSS лента получена")
return feed.Items
}
func CreatePost(mastoClient mastodon.Client, toot mastodon.Toot) {
_, err := mastoClient.PostStatus(context.Background(), &toot)
if err != nil {
log.Println(err)
}
}
func PicBytesArray(picturesArray []string) [][]byte {
var picturesBytes [][]byte
for _, picture := range picturesArray {
resp, err := http.Get(picture)
if err != nil {
log.Println(err)
return picturesBytes
}
defer resp.Body.Close()
picBytes, err := io.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return picturesBytes
}
picturesBytes = append(picturesBytes, picBytes)
}
return picturesBytes
}
func UploadPictures(mastoClient mastodon.Client, filesBytes [][]byte) []*mastodon.Attachment {
var attachments []*mastodon.Attachment
for _, file := range filesBytes {
att, err := mastoClient.UploadMediaFromBytes(context.Background(), file)
if err != nil {
log.Println(err)
return attachments
}
attachments = append(attachments, att)
}
return attachments
}
func CreateToot(mastoClient mastodon.Client, newsDesc *gofeed.Item, sensitive bool) (mastodon.Toot, error) {
var imgArray []string
var attachments []*mastodon.Attachment
var tootText string = fmt.Sprintf("src: %s\n\n", newsDesc.Link)
toot := mastodon.Toot{
Visibility: "unlisted",
Sensitive: sensitive,
}
uString := html.UnescapeString(newsDesc.Description)
pHtml, err := html.Parse(strings.NewReader(uString))
if err != nil {
return mastodon.Toot{}, err
}
for n := range pHtml.Descendants() {
if n.Type != html.ElementNode {
tootText += (n.Data)
}
if n.Type == html.ElementNode {
switch n.Data {
case "img":
for _, attr := range n.Attr {
if attr.Key == "src" {
imgArray = append(imgArray, attr.Val)
}
}
case "br", "p":
tootText += "\n"
}
}
}
if len(imgArray) != 0 {
attachments = UploadPictures(mastoClient, PicBytesArray(imgArray))
}
toot.Status = tootText
for _, attach := range attachments {
toot.MediaIDs = append(toot.MediaIDs, attach.ID)
}
return toot, nil
}