Skip to content
This repository has been archived by the owner on Oct 19, 2023. It is now read-only.

Commit

Permalink
save some intermediate work
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrei Kurilov committed Mar 12, 2023
1 parent fd82f5b commit 22899c8
Show file tree
Hide file tree
Showing 22 changed files with 720 additions and 70 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*.dll
*.so
*.dylib
producer-rss
./producer-rss
cover.tmp

# Test binary, built with `go test -c`
Expand Down
10 changes: 10 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
FROM golang:1.20.2-alpine3.17 AS builder
WORKDIR /go/src/producer-rss
COPY . .
RUN \
apk add protoc protobuf-dev make git && \
make build

FROM alpine:3.17.0
COPY --from=builder /go/src/producer-rss/producer-rss /bin/producer-rss
ENTRYPOINT ["/bin/producer-rss", "/etc/feeds.txt"]
3 changes: 1 addition & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ run: docker
docker run \
-d \
--name awakari-producer-rss \
-p 8080:8080 \
--expose 8080 \
--volume $(shell pwd)/config/feeds.txt:/etc/feeds.txt \
awakari/producer-rss

staging: docker
Expand Down
Empty file added README.md
Empty file.
11 changes: 10 additions & 1 deletion api/grpc/resolver/logging.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,16 @@ func NewLoggingMiddleware(svc Service, log *slog.Logger) Service {

func (lm loggingMiddleware) Submit(ctx context.Context, msg *event.Event) (err error) {
defer func() {
lm.log.Debug(fmt.Sprintf("resolver.Submit(msg.Id=%s): %s", msg.ID(), err))
lm.log.Debug(
fmt.Sprintf(
"resolver.Submit(msg(id: \"%s\", source: \"%s\" subject: \"%s\" time: %s)): %s",
msg.ID(),
msg.Source(),
msg.Subject(),
msg.Time(),
err,
),
)
}()
return lm.svc.Submit(ctx, msg)
}
27 changes: 19 additions & 8 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,26 @@ type Config struct {
Uri string `envconfig:"API_RESOLVER_URI" default:"resolver:8080" required:"true"`
}
}
Feed FeedConfig
Message struct {
Metadata MessageMetadataConfig
Feed FeedConfig
Log struct {
Level int `envconfig:"LOG_LEVEL" default:"-4" required:"true"`
}
Message MessageConfig
}

type FeedConfig struct {
UpdateInterval struct {
Min time.Duration `envconfig:"FEED_UPDATE_INTERVAL_MIN" default:"10s" required:"true"`
Max time.Duration `envconfig:"FEED_UPDATE_INTERVAL_MAX" default:"24h" required:"true"`
}
UpdateIntervalMin time.Duration `envconfig:"FEED_UPDATE_INTERVAL_MIN" default:"1s" required:"true"`
UpdateIntervalMax time.Duration `envconfig:"FEED_UPDATE_INTERVAL_MAX" default:"1m" required:"true"`
Urls []string // to be set externally
UserAgent string `envconfig:"FEED_USER_AGENT" default:"awakari-producer-rss/0.0.1" required:"true"`
}

type MessageConfig struct {
Metadata MetadataConfig
Content ContentConfig
}

type MessageMetadataConfig struct {
type MetadataConfig struct {
//
KeyFeedCategories string `envconfig:"MSG_MD_KEY_FEED_CATEGORIES" default:"feedcategories" required:"true"`
KeyFeedDescription string `envconfig:"MSG_MD_KEY_FEED_DESCRIPTION" default:"feeddescription" required:"true"`
Expand All @@ -34,6 +40,7 @@ type MessageMetadataConfig struct {
//
KeyAuthor string `envconfig:"MSG_MD_KEY_AUTHOR" default:"author" required:"true"`
KeyCategories string `envconfig:"MSG_MD_KEY_CATEGORIES" default:"categories" required:"true"`
KeyGuid string `envconfig:"MSG_MD_KEY_GUID" default:"rssitemguid" required:"true"`
KeyImageTitle string `envconfig:"MSG_MD_KEY_IMAGE_TITLE" default:"imagetitle" required:"true"`
KeyImageUrl string `envconfig:"MSG_MD_KEY_IMAGE_URL" default:"imageurl" required:"true"`
KeyLanguage string `envconfig:"MSG_MD_KEY_LANGUAGE" default:"language" required:"true"`
Expand All @@ -43,6 +50,10 @@ type MessageMetadataConfig struct {
SpecVersion string `envconfig:"MSG_MD_SPEC_VERSION" default:"1.0" required:"true"`
}

type ContentConfig struct {
Type string `envconfig:"MSG_CONTENT_TYPE" default:"text/html" required:"true"`
}

func NewConfigFromEnv() (cfg Config, err error) {
err = envconfig.Process("", &cfg)
return
Expand Down
27 changes: 27 additions & 0 deletions config/feeds.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
http://export.arxiv.org/rss/cs
http://export.arxiv.org/rss/econ
http://export.arxiv.org/rss/eess
http://export.arxiv.org/rss/math
http://export.arxiv.org/rss/astro-ph
http://export.arxiv.org/rss/cond-mat
http://export.arxiv.org/rss/gr-qc
http://export.arxiv.org/rss/hep-ex
http://export.arxiv.org/rss/hep-lat
http://export.arxiv.org/rss/hep-ph
http://export.arxiv.org/rss/hep-th
http://export.arxiv.org/rss/math-ph
http://export.arxiv.org/rss/nlin
http://export.arxiv.org/rss/nucl-ex
http://export.arxiv.org/rss/nucl-th
http://export.arxiv.org/rss/physics
http://export.arxiv.org/rss/quant-ph
http://export.arxiv.org/rss/q-bio
http://export.arxiv.org/rss/q-fin
http://export.arxiv.org/rss/stat
http://feeds.bbci.co.uk/news/rss.xml
https://lenta.ru/rss/news
https://feeds.yle.fi/uutiset/v1/majorHeadlines/YLE_UUTISET.rss
https://techcrunch.com/feed/
http://feeds.mashable.com/mashable
https://gizmodo.com/rss
https://hnrss.org/newest
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ require (
github.com/SlyMarbo/rss v1.0.5
github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.13.0
github.com/cloudevents/sdk-go/v2 v2.13.0
github.com/google/uuid v1.3.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/stretchr/testify v1.8.0
github.com/stretchr/testify v1.8.1
golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0
google.golang.org/grpc v1.53.0
google.golang.org/protobuf v1.29.0
Expand All @@ -17,9 +18,9 @@ require (
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/json-iterator/go v1.1.10 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sys v0.4.0 // indirect
Expand Down
23 changes: 23 additions & 0 deletions helm/producer-rss/.helmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
24 changes: 24 additions & 0 deletions helm/producer-rss/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: v2
name: producer-rss
description: A Helm chart for Awakari producer-rss

# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application

# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
22 changes: 22 additions & 0 deletions helm/producer-rss/templates/NOTES.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "producer-rss.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "producer-rss.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "producer-rss.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "producer-rss.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
62 changes: 62 additions & 0 deletions helm/producer-rss/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "producer-rss.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "producer-rss.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "producer-rss.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Common labels
*/}}
{{- define "producer-rss.labels" -}}
helm.sh/chart: {{ include "producer-rss.chart" . }}
{{ include "producer-rss.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}

{{/*
Selector labels
*/}}
{{- define "producer-rss.selectorLabels" -}}
app.kubernetes.io/name: {{ include "producer-rss.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}

{{/*
Create the name of the service account to use
*/}}
{{- define "producer-rss.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "producer-rss.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
94 changes: 94 additions & 0 deletions helm/producer-rss/templates/deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "producer-rss.fullname" . }}
labels:
{{- include "producer-rss.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "producer-rss.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "producer-rss.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "producer-rss.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
env:
- name: API_PORT
value: "{{ .Values.service.port }}"
- name: API_CONDITIONS_KIWI_TREE_COMPLETE_URI
value: {{ .Values.api.conditions.kiwi.tree.completeUri }}
- name: API_CONDITIONS_KIWI_TREE_PARTIAL_URI
value: {{ .Values.api.conditions.kiwi.tree.partialUri }}
- name: API_CONDITIONS_KIWI_BATCH_SIZE
value: "{{ .Values.api.conditions.kiwi.batchSize }}"
- name: API_SUBSCRIPTIONS_URI
value: {{ .Values.api.subscriptions.uri }}
- name: API_SUBSCRIPTIONS_BATCH_SIZE
value: "{{ .Values.api.subscriptions.batchSize }}"
- name: API_MATCHES_URI
value: {{ .Values.api.matches.uri }}
- name: API_ROUTER_URI
value: {{ .Values.api.router.uri }}
- name: LOG_LEVEL
value: "{{ .Values.log.level }}"
- name: QUEUE_BATCH_SIZE
value: "{{ .Values.queue.batchSize }}"
- name: QUEUE_FALLBACK_ENABLED
value: "{{ .Values.queue.fallback.enabled }}"
- name: QUEUE_FALLBACK_SUFFIX
value: "{{ .Values.queue.fallback.suffix }}"
- name: QUEUE_LIMIT
value: "{{ .Values.queue.limit }}"
- name: QUEUE_NAME
value: "{{ .Release.Name }}"
- name: QUEUE_SLEEP_ON_EMPTY_MILLIS
value: "{{ .Values.queue.sleepOn.emptyMillis }}"
- name: QUEUE_SLEEP_ON_ERROR_MILLIS
value: "{{ .Values.queue.sleepOn.errorMillis }}"
- name: QUEUE_URI
value: "{{ .Values.queue.uri }}"
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: grpc
containerPort: {{ .Values.service.port }}
protocol: TCP
livenessProbe:
grpc:
port: {{ .Values.service.port }}
readinessProbe:
grpc:
port: {{ .Values.service.port }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
Loading

0 comments on commit 22899c8

Please sign in to comment.