Greetings, dear readers. In this article, I would like to talk about my experience in building an internal network that is not tied to office equipment and functioning under the only condition that the Internet is available. With the addition of vpnclient feedback and network access control for each client to the server settings. And manage all this from one place via a web interface or a convenient GUI.

vpn -, , . , :

WiFi
VPN
vpn
vpn
ACL
ACL GUI
VPN

( Layer 3)
( ELK )

. , . , , .

. -, . , .

, vpn Ubuntu 18.04.5 LTS.

vpn WireGuard, . , . , . udp . 20%.

firewall iptables Shorewall. Shorewall .

Windows TightVNC, msi . , , jpeg /. . VNC.

, / GitLab CI Pipelines. / git . - .

Fluentd / Filebeat Elasticsearch.

, . .

wireguard Ubuntu 18.04.5 LTS

Ubuntu ≥ 18.04

sudo apt install wireguard

wg genkey | sudo tee /etc/wireguard/privatekey | wg pubkey | sudo tee /etc/wireguard/publickey

/etc/wireguard/

wg0.conf

sudo nano /etc/wireguard/wg0.conf

[Interface]
Address = 192.168.30.1/24       <-   
SaveConfig = true
ListenPort = 5505               <-   
PrivateKey = SERVER_PRIVATE_KEY

2 :

PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -t nat -A POSTROUTING -o ens3 -j MASQUERADE
PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -t nat -D POSTROUTING -o ens3 -j MASQUERADE

PostUp PostDown NAT. ip , , .

wg0, .

ip_forward

sudo nano /etc/sysctl.conf

#     
net.ipv4.ip_forward=1

, ip wireguard.

sudo sysctl -p

ufw

systemctl disable ufw

iptables

iptables -F
iptables -X
iptables -t nat -F
iptables -t nat -X
iptables -t mangle -F
iptables -t mangle -X
iptables -P INPUT ACCEPT
iptables -P FORWARD ACCEPT
iptables -P OUTPUT ACCEPT

sudo wg-quick up wg0

sudo wg

wg

sudo systemctl enable wg-quick@wg0

wg0.conf /etc/wireguard/

cp /etc/wireguard/wg0.conf /etc/wireguard/wg0.sempl

[Interface].

/opt

mkdir /opt/git
mkdir /opt/git/wg

wg0.conf /opt/git/wg

cp /etc/wireguard/wg0.conf /opt/git/wg/wg0.conf

ln -sf /opt/git/wg/wg0.conf /etc/wireguard/wg0.conf

? wg0.sempl , CI .gitlab-ci.yml . /opt/git/wg , /etc/wireguard. 2 , [Interface] [Peer] , gitlab.

udp: 5505 . - 192.168.30.0/24 . wg0.conf 192.168.30.1 , . . 192.168.30.0/24 ipsec 2 , .

Address = 192.168.30.X/24
DNS = 10.15.1.10, 10.16.1.252

[Peer]
PublicKey = <server_public_key>
AllowedIPs = 0.0.0.0/0
Endpoint = 1.2.3.4:5505

AllowedIPs 0.0.0.0/0, VPN. .

, .

Shorewall

shorewall

apt update

apt install -y shorewall

shorewall.conf . .

, shorewall.conf

nano /etc/shorewall/shorewall.conf

STARTUP_ENABLED=Yes
LOG_LEVEL="info(tcp_options,tcp_sequence,macdecode,ip_options)"
BLACKLIST_LOG_LEVEL="$LOG_LEVEL"
INVALID_LOG_LEVEL="$LOG_LEVEL"
LOG_MARTIANS=Yes
LOG_VERBOSITY=2
LOGALLNEW="$LOG_LEVEL"
LOGFILE=/opt/logs/shorewall/firewall.log
LOGFORMAT="ip-tables %s %s "
LOGTAGONLY=No

/opt . LOGFORMAT iptables /opt/logs/shorewall/firewall.log.

mkdir /opt/logs
mkdir /opt/logs/shorewall
touch /opt/logs/shorewall/firewall.log

iptables. ip-tables

nano /etc/rsyslog.d/10-my_iptables.conf

# Log kernel generated iptables log messages to file
:msg,contains,"ip-tables" /opt/logs/shorewall/firewall.log
& ~

, /. .

service rsyslog restart

shorewall .

interfaces,

nano /etc/shorewall/interfaces

?FORMAT 2
###############################################################################
#ZONE   INTERFACE       OPTIONS
lan     eth0            tcpflags,nosmurfs,routefilter,logmartians
wg      wg0             tcpflags,nosmurfs,routefilter,logmartians
#LAST LINE -- ADD YOUR ENTRIES BEFORE THIS ONE -- DO NOT REMOVE

nano /etc/shorewall/params.mgmt

# protocols
ALL_T_U=tcp,udp
# all services, networks and subnets
AD_DS=10.15.1.10,10.17.2.2
IPA=10.16.1.252
DNS_INT=10.15.1.10,10.16.1.252,192.168.0.253
KASPER=10.15.1.55
####
NET_OFFICE=10.15.1.0/24
NET_OFFICE_PRINTERS=10.15.14.0/24
##
NET_CLOUD_PROD=172.16.0.0/20,172.16.16.0/20,172.16.32.0/20
NET_CLOUD_DEV=192.168.128.0/24,192.168.1.0/24
####
VPN_01=192.168.30.0/24
VPN_02=192.168.40.0/24
####
ADM_IP=10.17.1.9
ADM_IP_VPN=192.168.30.3,192.168.40.3
VNC_SERVERS=10.15.1.10
###END###

, , , free ipa, DNS, VPN, ip , .

nano /etc/shorewall/params

INCLUDE params.mgmt

nano /etc/shorewall/policy

##
#SOURCE      DEST           POLICY      LOGLEVEL     LIMIT
$FW          lan            ACCEPT       $LOG_LEVEL
$FW          wg             DROP         $LOG_LEVEL
wg           $FW            DROP         $LOG_LEVEL
# THE FOLOWING POLICY MUST BE LAST
all          all            REJECT       $LOG_LEVEL
##

. $FW , .. wg . .

nano /etc/shorewall/zones

###############################################################################
#ZONE   TYPE            OPTIONS         IN                      OUT
#                                       OPTIONS                 OPTIONS
FW      firewall
lan     ipv4
wg      ipv4
#LAST LINE - ADD YOUR ENTRIES ABOVE THIS ONE - DO NOT REMOVE

nano /etc/shorewall/services.mgmt 

# add default rules for all connections
# AD DS & LDAP
ACCEPT   lan:$AD_DS         wg                  $ALL_T_U    42
ACCEPT   wg                 lan:$AD_DS          $ALL_T_U    42
ACCEPT   lan:$AD_DS         wg                  $ALL_T_U    88
ACCEPT   wg                 lan:$AD_DS          $ALL_T_U    88
ACCEPT   lan:$AD_DS         wg                  $ALL_T_U    135
ACCEPT   wg                 lan:$AD_DS          $ALL_T_U    135
.....

# Free IPA ports
ACCEPT   lan:$IPA           wg                  $ALL_T_U    88
.....

# internal DNS
ACCEPT   lan:$DNS_INT       wg                  $ALL_T_U    53
ACCEPT   wg                 lan:$DNS_INT        $ALL_T_U    53
# kastersky
ACCEPT   lan:$KASPER        wg                  $ALL_T_U    13000
ACCEPT   wg                 lan:$KASPER         $ALL_T_U    13000
.....

# admin all access
ACCEPT   lan:$ADM_IP        wg  
ACCEPT   wg:$ADM_IP_VPN     wg
ACCEPT   lan:$ADM_IP_VPN    wg
# vnc
ACCEPT   lan:$VNC_SERVERS   wg                  tcp         7900
#
###END###

. .

vpn

nano /etc/shorewall/networks.mgmt 

# drop internal networks to clients and from client
DROP     lan:$NET_OFFICE               wg
DROP     lan:$NET_OFFICE_PRINTERS      wg
DROP     lan:$NET_CLOUD_PROD           wg
DROP     lan:$NET_CLOUD_DEV            wg
DROP     wg                            lan:$NET_OFFICE            
DROP     wg                            lan:$NET_OFFICE_PRINTERS   
DROP     wg                            lan:$NET_CLOUD_PROD           
DROP     wg                            lan:$NET_CLOUD_DEV            
# wireguard networks
DROP     lan:$VPN_01                   wg
DROP     lan:$VPN_02                   wg
DROP     wg                            lan:$VPN_01
DROP     wg                            lan:$VPN_02
DROP     wg:$VPN_01                    wg
DROP     wg:$VPN_02                    wg
DROP     wg                            wg:$VPN_01
DROP     wg                            wg:$VPN_02
#
###END###

, .. Shorewall , 2 vpn / (, vpn)

mkdir /etc/shorewall/rules_internet.d
mkdir /etc/shorewall/rules_networks.d

.rule, <ip_username>, <ip_username>.rule. 2 .

nano /etc/shorewall/rules_internet.d/192.168.30.2_tst-client.rule 

###########################################################################################
#ACTION         SOURCE            DEST            PROTO   DEST    SOURCE          ORIGINAL        RATE            USER/   MARK
#                                                         PORT    PORT(S)         DEST            LIMIT           GROUP
ACCEPT          wg:192.168.30.2   lan                       
#
###END###

( )

nano /etc/shorewall/rules_networks.d/192.168.30.2_tst-client.rule 

###########################################################################################
#ACTION           SOURCE                     DEST                        PROTO   DEST    SOURCE          ORIGINAL        RATE            USER/   MARK
#                                                                                PORT    PORT(S)         DEST            LIMIT           GROUP
ACCEPT           wg:192.168.30.2            lan:$NET_OFFICE
DROP             wg:192.168.30.2            lan:10.15.1.69

ACCEPT           wg:192.168.30.2            lan:10.16.1.252              tcp     80
#
###END###

DROP, , rules.

, rules

nano /etc/shorewall/rules

###########################################################################################
#ACTION          SOURCE         DEST        PROTO   DEST    SOURCE      ORIGINAL    RATE        USER/   MARK
#                                                   PORT    PORT(S)     DEST        LIMIT       GROUP
ACCEPT           lan             $FW
ACCEPT           wg              $FW        icmp
ACCEPT           wg              $FW        tcp     22
#
#####
# add services
INCLUDE services.mgmt
#
####
# add internal hosts/networks to clients
SHELL cat /etc/shorewall/rules_networks.d/*.rule
#
#####
# drop all internal networks to wireguard from clients and to clients
INCLUDE networks.mgmt
#
####
# add internet to clients
SHELL cat /etc/shorewall/rules_internet.d/*.rule
#
###END###

/ /
vpn
( )

/etc/shorewall/
├── conntrack
├── interfaces
├── networks.mgmt
├── params
├── params.mgmt
├── policy
├── README.MD
├── rules
├── rules_internet.d
│   └── 192.168.30.2_tst-client.rule
├── rules_networks.d
│   └── 192.168.30.2_tst-client.rule
├── services.mgmt
├── shorewall.conf
└── zones

shorewall reload

"", shorewall /etc/shorewall/ - , .. rules_internet.d/rules_networks.d iptables. gitlab.

Wireguard, Shorewall vpn . . vpn , vpn , . vpn ip , ADM_IP ADM_IP_VPN.

, :)

VPN CI Pipelines GitLab

selfhosted gitlab , , - docker-compose.

, vpn-01

 vpn-01
 ├── .gitlab-ci.yml
 ├── README.md
 ├── shorewall
 │   ├── networks.mgmt
 │   ├── params.mgmt
 │   ├── README.MD
 │   ├── rules_internet.d
 │   │   └── 192.168.30.2_tst-client.rule
 │   ├── rules_networks.d
 │   │   └── 192.168.30.2_tst-client.rule
 │   └── services.mgmt
 └── wireguard
     ├── README.MD
     └── wg0.conf

, shorewall gitlab. README.MD , .

wg0.conf .

[Peer]
PublicKey = <client_public_key>
AllowedIPs = 192.168.30.2/32

, shorewall 1 . gitlab-runner, Deploy Token read_repository .gitlab-ci.yml.

gitlab-runner vpn

apt install gitlab-runner -y

token Settings -> CI/CD -> Runners

sudo gitlab-runner register

url , , executor: shell tag: vpn-01, - /etc/gitlab-runner/config.toml.

deploy token, Settings -> Repository -> Deploy Tokens read_repository

vpn gitlab-runner sudo

#1
sudo usermod -a -G sudo gitlab-runner

#2
nano /etc/sudoers.d/gitlab-runner

#3
gitlab-runner  ALL=(ALL) NOPASSWD:/usr/bin/wg-quick,/usr/bin/git,/sbin/shorewall,/bin/cp,/bin/rm,/bin/cat,/bin/touch,/bin/chmod

/opt/git/

.gitlab-ci.yml

stages:
   - all

task-all:
  stage: all
  script: 
    - sudo /bin/cp -f /etc/wireguard/wg0.sempl /opt/git/wg/wg0.conf
    - sudo /usr/bin/wg-quick up wg0 || if [ $? -ne 0 ]; then echo "wg0 is up"; fi
    - sudo /usr/bin/wg-quick down wg0 
    - sudo /bin/rm -rf /opt/git/vpn-01 
    - cd /opt/git 
    - sudo /usr/bin/git clone https://gitlab+deploy-token:<token>@gitlab.company.net/infra/vpn-01.git 
    - sudo /bin/rm -rf /etc/shorewall/rules_networks.d/*
    - sudo /bin/rm -rf /etc/shorewall/rules_internet.d/*
    - sudo /bin/cp -rf /opt/git/wgvpn-02/shorewall/* /etc/shorewall/ 
    - sudo /bin/rm -rf /opt/git/wg/wg0.conf 
    - sudo /bin/cp /etc/wireguard/wg0.sempl /opt/git/wg/wg0.conf
    - sudo /bin/chmod 0666 /opt/git/wg/wg0.conf
    - sudo /bin/cat /opt/git/vpn-01/wireguard/wg0.conf >> /opt/git/wg/wg0.conf 
    - sudo /usr/bin/wg-quick up wg0 
    - sudo /sbin/shorewall reload 
    - sudo /usr/bin/wg-quick down wg0 
    - sleep 60 && sudo /usr/bin/wg-quick up wg0
  tags:
    - 'vpn-01'
  allow_failure: true
  when: manual

wg0, wg0.conf /
,
wg0 , wg0.conf
vpn-01
, shorewall
wg0.conf 2 , ,
wg0 shorewall , , wg0
60 , , ACCEPT ,

CI/CD -> Pipelines, Commit .

Merge Requests, Pull Requests.

, )

, GitLab, Elasticsearch Kibana. Elasticsearch.

vpn

nano /etc/security/limits.conf

root soft nofile 65536
root hard nofile 65536
* soft nofile 65536
* hard nofile 65536

vpn Fluentd

# td-agent 4
curl -L https://toolbelt.treasuredata.com/sh/install-ubuntu-bionic-td-agent4.sh | sh

/etc/td-agent/td-agent.conf ( )

'<source>
  @type tail
  path /opt/logs/shorewall/firewall.log
  pos_file /var/log/td-agent/pos-firewall.pos
  <parse>
    @type syslog
  </parse>
  tag firewall.raw
</source>

<match firewall.raw.**>
    @type elasticsearch
    host <server_ip>
    port <server_port>
    logstash_format true
    logstash_prefix infra-vpn-01
    flush_interval 10s
    flush_thread_count 2
</match>

syslog, syslog

Filebeat . .

#1
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -

#2
sudo apt-get install apt-transport-https

#3
sudo apt-get update && sudo apt-get install filebeat

#4
sudo systemctl enable filebeat

iptables

filebeat modules enable iptables

nano /etc/filebeat/modules.d/iptables.yml 

- module: iptables
  log:
    enabled: true
    var.input: "file"
    var.paths: ["/opt/logs/shorewall/firewall.log"]

/etc/filebeat/filebeat.yml Elasticsearch

output.elasticsearch:
  hosts: ["<ip_address>:9200"]

Elasticsearch Elasticsearch 7000 RAM — 10 Gb vCPU — 6.

, /etc/security/limits.conf , /etc/elasticsearch/jvm.options -Xms RAM.

elasticsearch   -      nofile         200000
elasticsearch    memlock 200000

* soft nofile 265536
* hard nofile 265536

/etc/elasticsearch/elasticsearch.yml

cluster.max_shards_per_node: 15000
xpack.ml.max_open_jobs: 100
cluster.routing.allocation.node_initial_primaries_recoveries: 10
thread_pool.search.queue_size: 100000
thread_pool.search.max_queue_size: 150000
thread_pool.search.size: 35
thread_pool.search.auto_queue_frame_size: 10000

"type" : "file_system_exception",
   "reason" : "/mnt/elk/data/nodes/0/indices/SNMMbQeLRlW0y4Vi_V9L1Q/3/_state: Too many open files"

, /etc/systemd/system/elasticsearch.service.d/elasticsearch.conf /etc/systemd/system/multi-user.target.wants/elasticsearch.service

LimitNOFILE=200000
LimitNPROC=4096
LimitAS=infinity
LimitFSIZE=infinity

/ 7000 .

, , ? , , , 2 .

, , .

Sources used:

WireGuard - https://www.wireguard.com

Shorewall - https://shorewall.org

TightVNC - https://www.tightvnc.com/download.php

GitLab runners - https://docs.gitlab.com/ee/ci/runners/

Linux firewall log format - http://www.stearns.org/doc/william_stearns_gcia.html#iptablesformat

Regular Expression Checker - https://rubular.com/

Fluentd - https://docs.fluentd.org/installation/before-install

Filebeat Iptables module - https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-iptables.html

WireGuard without NAT, intranet and loopback clients

wireguard Ubuntu 18.04.5 LTS

Shorewall

VPN CI Pipelines GitLab

Sources used:

More articles: