Skip to content

Commit

Permalink
Adds 'collect' command (#3)
Browse files Browse the repository at this point in the history 
- adds new dependencies for db (sqlite) and web (gin)
- stores timeline results in db
- adds flag to specify the database file
- adds query to report results
- root command sets a default config/server name
- updates readme
- updates goreleaser.yml to support FTS extension
- rewrites '--server' flag to allow list of servers
  • Loading branch information
@ivan3bx
ivan3bx authored Nov 21, 2022
1 parent 5650cb8 commit e04e7df
Show file tree
Hide file tree
Showing 16 changed files with 893 additions and 27 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
.vscode
dist/
*.db
5 changes: 4 additions & 1 deletion .goreleaser.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ builds:
main: ./main.go
binary: "proma"
env:
- CGO_ENABLED=0
# - CGO_ENABLED=1
ldflags:
- -s -w
- -X main.version={{.Version}}
flags:
# needed to enable sqlite's FTS extension
# - --tags=fts5
goos:
- linux
- darwin
Expand Down
55 changes: 49 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,32 @@
# proma

```text
CLI tool for extracting data from the Mastodon API.
proma is a CLI tool for querying data via the Mastodon API.
Usage:
proma [command]
Available Commands:
auth Authenticate with a Mastodon server.
collect Collects and aggregates tagged posts
links Extract links from any saved bookmarks
help Help about any command
Flags:
-c, --config string config file (default is $HOME/.proma.json)
-h, --help help for proma
-s, --server string server name (default "mastodon.social")
-v, --verbose verbose mode
-c, --config string config file (default is $HOME/.proma.json)
-h, --help help for proma
-s, --servers strings server names to check (default [mastodon.social])
-v, --verbose verbose mode
Use "proma [command] --help" for more information about a command.
```

## Example: extracting links from saved bookmarks
## Examples

### Extracting links from saved bookmarks

```bash
# creates a list of embedded links from bookmarked posts
./proma links --limit 2
```

Expand All @@ -41,3 +46,41 @@ Use "proma [command] --help" for more information about a command.
}
]
```

### Collecting posts by hashtag across multiple instances

```bash
# collects posts containing hashtags 'streetphotography' OR 'london'
./proma collect -t streetphotography,london -s mastodon.cloud,indieweb.social,social.linux.pizza
collecting from server: https://mastodon.cloud
collecting from server: https://indieweb.social
collecting from server: https://social.linux.pizza
...
```

```json
[
{
"uri": "https://photog.social/users/keirgravil/statuses/109377529017885305",
"lang": "en",
"content": "\u003cp\u003eCute little mushroom I spotted whilst out walking a section...",
"tag_list": [
"photography",
"london",
"autumn"
],
"created_at": "2022-11-20T18:24:03Z"
},
{
"uri": "https://mastodon.social/users/jesswade/statuses/109377446881326502",
"lang": "en",
"content": "...",
"tag_list": [
"london",
"urbanphotography"
],
"created_at": "2022-11-20T18:03:10Z"
}
...
]
```
12 changes: 11 additions & 1 deletion client/authorization.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@ import (
log "github.com/sirupsen/logrus"
)

// NewAnonymousClient returns a client capable of only returning data using
// public endpoints. Any authenticated calls through this client will fail.
func NewAnonymousClient(serverName string) *mastodon.Client {
return mastodon.NewClient(&mastodon.Config{
Server: serverURL(serverName),
})
}

// RegisterNewClient registers a new authenticated client by starting a local
// auth server, opening a browser and capturing client id & secret for this user.
func RegisterNewClient(serverName string) (*mastodon.Client, error) {
done := make(chan os.Signal, 1)

Expand Down Expand Up @@ -54,7 +64,7 @@ func RegisterNewClient(serverName string) (*mastodon.Client, error) {

app, err := mastodon.RegisterApp(context.Background(), &mastodon.AppConfig{
Server: serverURL(serverName),
ClientName: "Links From Bookmarks",
ClientName: "Proma for Mastodon",
Scopes: "read:bookmarks read:favourites",
Website: "https://github.com/ivan3bx/proma",
RedirectURIs: fmt.Sprintf("http://%s/auth", listenerHost),
Expand Down
30 changes: 30 additions & 0 deletions client/feed_processing.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package client

import (
"context"
"encoding/json"
"os"

"github.com/mattn/go-mastodon"
log "github.com/sirupsen/logrus"
)

type TagTimeline func(tag string) ([]*mastodon.Status, error)

// ServerFeed returns a TagTimeline using the provided client.
func ServerFeed(ctx context.Context, client *mastodon.Client) TagTimeline {
return func(tag string) ([]*mastodon.Status, error) {
log.Debugf("fetching timeline for tag: '%v'", tag)
return client.GetTimelineHashtag(ctx, tag, false, nil)
}
}

// FileFeed returns a TagTimeline using the provided filename source
func FileFeed(filename string) TagTimeline {
return func(tag string) ([]*mastodon.Status, error) {
f, _ := os.Open(filename)
data := []*mastodon.Status{}
err := json.NewDecoder(f).Decode(&data)
return data, err
}
}
34 changes: 34 additions & 0 deletions client/feed_processing_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package client

import (
"testing"

"github.com/mattn/go-mastodon"
)

func TestFetchItems(t *testing.T) {
testCases := []struct {
name string
input string
assertFunc func(*testing.T, []*mastodon.Status, error)
}{
{
name: "tagged items",
input: "testfiles/tag_timeline.json",
assertFunc: func(t *testing.T, items []*mastodon.Status, err error) {
if err != nil {
t.Error(err)
}
if len(items) < 2 {
t.Error("Expected 2 items, was ", len(items))
}
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
items, err := FileFeed(tc.input)("ignore_")
tc.assertFunc(t, items, err)
})
}
}
174 changes: 174 additions & 0 deletions client/testfiles/tag_timeline.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
[
{
"id": "109286874256020359",
"created_at": "2022-11-04T18:09:21.580Z",
"in_reply_to_id": null,
"in_reply_to_account_id": null,
"sensitive": false,
"spoiler_text": "",
"visibility": "public",
"language": "en",
"uri": "https://mastodon.social/users/HeathAllyn/statuses/109286874256020359",
"url": "https://mastodon.social/@HeathAllyn/109286874256020359",
"replies_count": 0,
"reblogs_count": 0,
"favourites_count": 0,
"edited_at": null,
"content": "<p>Downside: no internet still and I don’t see any outages so I do t think it’s an area-wide problem but I’m housesitting so it’s not my internet. I keep going “Oh maybe I’ll…oh no, that requires internet.”</p><p>Upside: I’ll be gone most of the day to an out of town gig. </p><p>Downside: I have to drive 2 hours there and 2 hours back in potential storms and then have a 7:40am call time for a music video shoot. <a href=\"https://mastodon.social/tags/internet\" class=\"mention hashtag\" rel=\"tag\">#<span>internet</span></a> <a href=\"https://mastodon.social/tags/outage\" class=\"mention hashtag\" rel=\"tag\">#<span>outage</span></a> <a href=\"https://mastodon.social/tags/musician\" class=\"mention hashtag\" rel=\"tag\">#<span>musician</span></a> <a href=\"https://mastodon.social/tags/actor\" class=\"mention hashtag\" rel=\"tag\">#<span>actor</span></a></p>",
"reblog": null,
"application": {
"name": "Metatext",
"website": "https://metabolist.org/metatext"
},
"account": {
"id": "109243907179961538",
"username": "HeathAllyn",
"acct": "HeathAllyn",
"display_name": "Heath Allyn",
"locked": false,
"bot": false,
"discoverable": true,
"group": false,
"created_at": "2022-10-28T00:00:00.000Z",
"note": "<p>Actor/Musician/Voiceover/Wizard/Geek/Shenanigangster/Jedi/Love Warrior. Wearer of many hats. Metaphorically. I almost never wear a literal hat. Almost. Lefty, liberal leanings so if that makes you &quot;HARUMPH&quot; then best move along.</p>",
"url": "https://mastodon.social/@HeathAllyn",
"avatar": "https://files.mastodon.social/accounts/avatars/109/243/907/179/961/538/original/09ae3560b69ae065.jpg",
"avatar_static": "https://files.mastodon.social/accounts/avatars/109/243/907/179/961/538/original/09ae3560b69ae065.jpg",
"header": "https://files.mastodon.social/accounts/headers/109/243/907/179/961/538/original/0607b7c557dcb182.png",
"header_static": "https://files.mastodon.social/accounts/headers/109/243/907/179/961/538/original/0607b7c557dcb182.png",
"followers_count": 40,
"following_count": 12,
"statuses_count": 144,
"last_status_at": "2022-11-15",
"noindex": false,
"emojis": [],
"fields": [
{
"name": "Music",
"value": "<a href=\"https://heathallyn.bandcamp.com\" target=\"_blank\" rel=\"nofollow noopener noreferrer me\"><span class=\"invisible\">https://</span><span class=\"\">heathallyn.bandcamp.com</span><span class=\"invisible\"></span></a>",
"verified_at": null
},
{
"name": "Silly Music",
"value": "<a href=\"https://shutupandlistentomysongs.com\" target=\"_blank\" rel=\"nofollow noopener noreferrer me\"><span class=\"invisible\">https://</span><span class=\"\">shutupandlistentomysongs.com</span><span class=\"invisible\"></span></a>",
"verified_at": "2022-11-02T01:11:17.492+00:00"
},
{
"name": "Instagram",
"value": "<a href=\"https://instagram.com/heathallyn\" target=\"_blank\" rel=\"nofollow noopener noreferrer me\"><span class=\"invisible\">https://</span><span class=\"\">instagram.com/heathallyn</span><span class=\"invisible\"></span></a>",
"verified_at": null
},
{
"name": "Website",
"value": "<a href=\"https://heathallyn.com\" target=\"_blank\" rel=\"nofollow noopener noreferrer me\"><span class=\"invisible\">https://</span><span class=\"\">heathallyn.com</span><span class=\"invisible\"></span></a>",
"verified_at": "2022-11-02T01:11:18.423+00:00"
}
]
},
"media_attachments": [],
"mentions": [],
"tags": [
{
"name": "internet",
"url": "https://mastodon.social/tags/internet"
},
{
"name": "outage",
"url": "https://mastodon.social/tags/outage"
},
{
"name": "musician",
"url": "https://mastodon.social/tags/musician"
},
{
"name": "actor",
"url": "https://mastodon.social/tags/actor"
}
],
"emojis": [],
"card": null,
"poll": null
},
{
"id": "109263727273144392",
"created_at": "2022-10-31T16:02:46.661Z",
"in_reply_to_id": "109263717307092988",
"in_reply_to_account_id": "108197211708251992",
"sensitive": false,
"spoiler_text": "",
"visibility": "public",
"language": "en",
"uri": "https://mastodon.social/users/MaaikeV/statuses/109263727273144392",
"url": "https://mastodon.social/@MaaikeV/109263727273144392",
"replies_count": 0,
"reblogs_count": 0,
"favourites_count": 1,
"edited_at": null,
"content": "<p>via @[email protected]<br />This is such a bad outage for <br /><span class=\"h-card\"><a href=\"https://mastodon.social/@instagram\" class=\"u-url mention\">@<span>instagram</span></a></span>. </p><p>The &quot;we suspended your account&quot; UX will put users into an emotionally vulnerable state, making them susceptible to phishing attacks.</p><p><a href=\"https://mastodon.social/tags/instagram\" class=\"mention hashtag\" rel=\"tag\">#<span>instagram</span></a> <br /><a href=\"https://mastodon.social/tags/outage\" class=\"mention hashtag\" rel=\"tag\">#<span>outage</span></a><br /><a href=\"https://mastodon.social/tags/phishing\" class=\"mention hashtag\" rel=\"tag\">#<span>phishing</span></a> <br /><a href=\"https://mastodon.social/tags/warning\" class=\"mention hashtag\" rel=\"tag\">#<span>warning</span></a></p>",
"reblog": null,
"application": {
"name": "Web",
"website": null
},
"account": {
"id": "108197211708251992",
"username": "MaaikeV",
"acct": "MaaikeV",
"display_name": "Maaike",
"locked": false,
"bot": false,
"discoverable": true,
"group": false,
"created_at": "2022-04-26T00:00:00.000Z",
"note": "<p>Social Psychologist, digital native, checks sources, loves irony. Maker of humans, breaker of stuff, hacker of both. Tries not to judge. Google-fu </p><p>Interested in the connection between <a href=\"https://mastodon.social/tags/infosec\" class=\"mention hashtag\" rel=\"tag\">#<span>infosec</span></a>, <a href=\"https://mastodon.social/tags/cyber\" class=\"mention hashtag\" rel=\"tag\">#<span>cyber</span></a>, <a href=\"https://mastodon.social/tags/users\" class=\"mention hashtag\" rel=\"tag\">#<span>users</span></a>, <a href=\"https://mastodon.social/tags/ui\" class=\"mention hashtag\" rel=\"tag\">#<span>ui</span></a> and <a href=\"https://mastodon.social/tags/training\" class=\"mention hashtag\" rel=\"tag\">#<span>training</span></a></p>",
"url": "https://mastodon.social/@MaaikeV",
"avatar": "https://files.mastodon.social/accounts/avatars/108/197/211/708/251/992/original/e9333ee04af164f9.jpeg",
"avatar_static": "https://files.mastodon.social/accounts/avatars/108/197/211/708/251/992/original/e9333ee04af164f9.jpeg",
"header": "https://files.mastodon.social/accounts/headers/108/197/211/708/251/992/original/a1da62c161f85f38.jpeg",
"header_static": "https://files.mastodon.social/accounts/headers/108/197/211/708/251/992/original/a1da62c161f85f38.jpeg",
"followers_count": 288,
"following_count": 325,
"statuses_count": 448,
"last_status_at": "2022-11-15",
"noindex": false,
"emojis": [],
"fields": [
{
"name": "Twitter",
"value": "@iktwiet",
"verified_at": null
}
]
},
"media_attachments": [],
"mentions": [
{
"id": "1632",
"username": "instagram",
"url": "https://mastodon.social/@instagram",
"acct": "instagram"
}
],
"tags": [
{
"name": "instagram",
"url": "https://mastodon.social/tags/instagram"
},
{
"name": "outage",
"url": "https://mastodon.social/tags/outage"
},
{
"name": "phishing",
"url": "https://mastodon.social/tags/phishing"
},
{
"name": "warning",
"url": "https://mastodon.social/tags/warning"
}
],
"emojis": [],
"card": null,
"poll": null
}
]
Loading

0 comments on commit e04e7df

Please sign in to comment.