大数据系列 | 使用Nginx作为数据采集接收端

1. 编译安装Nginx
2. 通过程序对Nginx上传日志

1. 编译安装Nginx

Nginx依赖包安装：

root@nginx:~# apt install -y vim lua-devel gcc gcc-c++ autoconf automake zlib zlib-devel openssl openssl-devel gcc zlib1g-dev libpcre3 libpcre3-dev  openssl libssl-dev

root@nginx:/opt# tar xf lua-nginx-module-0.10.13.tar.gz     // 网络上可以下载
root@nginx:/opt# tar xf ngx_devel_kit-0.3.0.tar.gz			// 网络上可以下载
root@nginx:/opt# git clone https://luajit.org/git/luajit.git
root@nginx:/opt/luajit# make PREFIX=/opt/LuaJIT
==== Building LuaJIT 2.1 ====
......(省略部分)......
OK        Successfully built LuaJIT
make[1]: Leaving directory '/opt/luajit/src'
==== Successfully built LuaJIT 2.1 ====root@nginx:/opt/luajit# make install PREFIX=/opt/LuaJIT
......(省略部分)......
==== Successfully installed LuaJIT 2.1.1710088188 to /opt/LuaJIT ====

root@nginx:/opt# vim /etc/profile
export LUAJIT_LIB=/opt/LuaJIT/lib
export LUAJIT_INC=/opt/LuaJIT/include/luajit-2.1
root@nginx:/opt# source /etc/profile

luajit安装官方手册：https://luajit.org/install.html

编译安装Nginx：

root@nginx:~# useradd -s /sbin/nologin -M nginx

root@nginx:~# tar xf nginx-1.16.1.tar.gz
root@nginx:~# mv /root/nginx-1.16.1 /root/nginx
root@nginx:~# cd nginx
root@nginx:~/nginx# ./configure  \
--prefix=/opt/nginx/ \
--sbin-path=/opt/nginx/sbin/nginx \
--conf-path=/opt/nginx/conf/nginx.conf \
--error-log-path=/opt/nginx/logs/error.log \
--http-log-path=/opt/nginx/logs/access.log \
--pid-path=/opt/nginx/logs/nginx.pid \
--lock-path=/opt/nginx/logs/nginx.lock \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_mp4_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--with-http_auth_request_module \
--with-stream --with-stream_ssl_module \
--add-module=/opt/ngx_devel_kit-0.3.0 \
--add-module=/opt/lua-nginx-module-0.10.13root@nginx:~# make && make install

Ubuntu22.04编译安装nginx时报错error: ‘ENGINE_free’ is deprecated: Since OpenSSL 3.0
加上：./configure --with-cc-opt="-Wno-error -Wno-deprecated-declarations"

[root@localhost ~]# /opt/nginx/sbin/nginx -V
nginx version: nginx/1.16.1
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-39) (GCC)
built with OpenSSL 1.0.2k-fips  26 Jan 2017
TLS SNI support enabled
configure arguments: --prefix=/opt/nginx/ --sbin-path=/opt/nginx/sbin/nginx --conf-path=/opt/nginx/conf/nginx.conf --error-log-path=/opt/nginx/logs/error.log --http-log-path=/opt/nginx/logs/access.log --pid-path=/opt/nginx/logs/nginx.pid --lock-path=/opt/nginx/logs/nginx.lock --user=nginx --group=nginx --with-http_ssl_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-http_auth_request_module --with-stream --with-stream_ssl_module --add-module=/opt/ngx_devel_kit-0.3.0 --add-module=/opt/lua-nginx-module-0.10.13

编辑nginx.conf配置文件：

root@nginx:~# cat /opt/nginx/conf/nginx.conf
user  nginx;
worker_processes  42;error_log  logs/error.log;pid        logs/nginx.pid;events {use epoll;multi_accept on;worker_connections  65535;
}http {include       mime.types;default_type  text/html;log_format  main  '$remote_addr - $remote_user [$time_local] "$request" ''$status $body_bytes_sent "$http_referer" ''"$http_user_agent" "$http_x_forwarded_for"';log_format  demo  escape=json 'ip=$remote_addr&$request_body';log_format  test  escape=json '$remote_addr | $http_x_forwarded_for |  $time_local | $request | $uri | ''$status | $body_bytes_sent | $bytes_sent | $gzip_ratio | $http_referer | ''$http_user_agent | $http_Authorization | $upstream_addr | $upstream_response_time | $upstream_status | $request_time | $request_body | $http_host';keepalive_timeout  120;keepalive_requests 10000;server_names_hash_bucket_size 128;client_header_buffer_size     32k;large_client_header_buffers 4 32k;fastcgi_buffers 32 8k;client_body_buffer_size 1024k;server_tokens off;gzip  on;gzip_http_version 1.0;gzip_min_length 1100;gzip_buffers    4 16k;gzip_comp_level 9;gzip_vary on;gzip_types  text/plain application/x-javascript application/javascript text/css application/xml text/javascript;output_buffers  1 32k;postpone_output 1460;add_header X-Cachei $upstream_cache_status;include /opt/nginx/conf/vhost/*.conf;}

编辑vhost文件：

root@nginx:~#  mkdir -p /opt/nginx/logs/test/

root@nginx:~# mkdir /opt/nginx/conf/vhost/
root@nginx:~# vim /opt/nginx/conf/vhost/172.26.160.14.conf
upstream wangzongyu {server 10.1.248.197:81 weight=10 max_fails=2 fail_timeout=10s;keepalive 300;
}server {listen       80;server_name  172.26.160.14;include    /opt/nginx/conf/proxy.conf;location / {proxy_pass http://wangzongyu/;content_by_lua 'ngx.header.content_type = "text/html";ngx.req.read_body()';proxy_http_version 1.1;proxy_set_header Connection "";access_log     /opt/nginx/logs/test/wangzongyu.log test;error_log      /opt/nginx/logs/error.wangzongyu.log;}location  = /ngx_status {stub_status on;access_log off;allow all;#deny all;}
}server {listen  81;location  = / {root /opt/nginx/html;index  index.html index.htm;access_log off;#return 200;}}

启动Nginx：

root@nginx:~# /opt/nginx/sbin/nginx -t
nginx: the configuration file /opt/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /opt/nginx/conf/nginx.conf test is successfulroot@nginx:~#  /opt/nginx/sbin/nginx

root@nginx:~# cat /opt/nginx/logs/test/wangzongyu.log
192.168.223.63 |  |  20/Mar/2024:08:37:53 +0000 | GET / HTTP/1.1 | / | 200 | 31 | 226 | 0.00 |  | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 |  |  |  |  | 0.000 |  | 172.26.160.14