코드 분석

main()

// nginx/src/core/nginx.c

int ngx_cdecl
main(int argc, char *const *argv)
{
		...
		if (ngx_process == NGX_PROCESS_SINGLE) {
        ngx_single_process_cycle(cycle);

    } else {
        ngx_master_process_cycle(cycle);
    }
		
		return 0;
}

• 일단 ngx_single_process_cycle() 로 가정.

ngx_single_process_cycle()

// nginx/src/os/unix/ngx_process_cycle.c

void
ngx_single_process_cycle(ngx_cycle_t *cycle)
{
		...
		for ( ;; ) {
        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "worker cycle");

        ngx_process_events_and_timers(cycle);
				...
    }
}

• for ( ;; ) 문 돌면서 ngx_process_events_and_timers() 호출

ngx_process_events_and_timers()

// nginx/src/event/ngx_event.c

void
ngx_process_events_and_timers(ngx_cycle_t *cycle)
{
		...
    (void) ngx_process_events(cycle, timer, flags);
		...
}

• 내부적으로 ngx_process_events() 호출, 해당 함수는 매크로 함수에 해당한다.

  // nginx/src/event/ngx_event.h
  
  #define ngx_process_events   ngx_event_actions.process_events

• ngx_event_actions는 어떤 녀석인가? (macOS X 기준 - kqueue 사용)

  // nginx/src/event/ngx_event.h
  
  typedef struct {
      ngx_int_t  (*add)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
      ngx_int_t  (*del)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
  
      ngx_int_t  (*enable)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
      ngx_int_t  (*disable)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
  
      ngx_int_t  (*add_conn)(ngx_connection_t *c);
      ngx_int_t  (*del_conn)(ngx_connection_t *c, ngx_uint_t flags);
  
      ngx_int_t  (*notify)(ngx_event_handler_pt handler);
  
      ngx_int_t  (*process_events)(ngx_cycle_t *cycle, ngx_msec_t timer,
                                   ngx_uint_t flags);
  
      ngx_int_t  (*init)(ngx_cycle_t *cycle, ngx_msec_t timer);
      void       (*done)(ngx_cycle_t *cycle);
  } ngx_event_actions_t;
  
  extern ngx_event_actions_t   ngx_event_actions;

  • 전역변수
  • struct ngx_event_actions_t 에 해당한다.
  • 위의 ngx_event_actions.process_events는 ngx_event_actions 내부의 ngx_int_t (*process_events)(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); 함수포인터에 해당한다.
• 그러면 그 등록된 함수포인터는 실제로는 어떤 함수인가?

  // nginx/src/event/ngx_event.h
  
  typedef struct {
      ngx_str_t              *name;
  
      void                 *(*create_conf)(ngx_cycle_t *cycle);
      char                 *(*init_conf)(ngx_cycle_t *cycle, void *conf);
  
      ngx_event_actions_t     actions;
  } ngx_event_module_t;

  // nginx/src/event/modules/ngx_kqueue_module.c
  
  static ngx_event_module_t  ngx_kqueue_module_ctx = {
      &kqueue_name,
      ngx_kqueue_create_conf,                /* create configuration */
      ngx_kqueue_init_conf,                  /* init configuration */
  
      {
          ngx_kqueue_add_event,              /* add an event */
          ngx_kqueue_del_event,              /* delete an event */
          ngx_kqueue_add_event,              /* enable an event */
          ngx_kqueue_del_event,              /* disable an event */
          NULL,                              /* add an connection */
          NULL,                              /* delete an connection */
  #ifdef EVFILT_USER
          ngx_kqueue_notify,                 /* trigger a notify */
  #else
          NULL,                              /* trigger a notify */
  #endif
          ngx_kqueue_process_events,         /* process the events */
          ngx_kqueue_init,                   /* init the events */
          ngx_kqueue_done                    /* done the events */
      }
  
  };
  
  static ngx_int_t
  ngx_kqueue_init(ngx_cycle_t *cycle, ngx_msec_t timer)
  {
  		...
      ngx_event_actions = ngx_kqueue_module_ctx.actions;
  
      return NGX_OK;
  }

  • 즉, ngx_event_actions 변수에는 ngx_kqueue_process_events() 함수가 들어가게 된다.
• 결론 : ngx_process_events() 는 사실 ngx_kqueue_process_events() 였던 것이다.

ngx_kqueue_process_events()

static ngx_int_t
ngx_kqueue_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
    ngx_uint_t flags)
{
    int               events, n;
    ngx_int_t         i, instance;
    ngx_uint_t        level;
    ngx_err_t         err;
    ngx_event_t      *ev;
    ngx_queue_t      *queue;
    struct timespec   ts, *tp;

		...

    events = kevent(ngx_kqueue, change_list, n, event_list, (int) nevents, tp);

		...

    for (i = 0; i < events; i++) {

        ngx_kqueue_dump_event(cycle->log, &event_list[i]);

        if (event_list[i].flags & EV_ERROR) {
            ngx_log_error(NGX_LOG_ALERT, cycle->log, event_list[i].data,
                          "kevent() error on %d filter:%d flags:%04Xd",
                          (int) event_list[i].ident, event_list[i].filter,
                          event_list[i].flags);
            continue;
        }

				...

        ev = (ngx_event_t *) event_list[i].udata;

        switch (event_list[i].filter) {

        case EVFILT_READ:
        case EVFILT_WRITE:

            instance = (uintptr_t) ev & 1;
            ev = (ngx_event_t *) ((uintptr_t) ev & (uintptr_t) ~1);

            if (ev->closed || ev->instance != instance) {

                /*
                 * the stale event from a file descriptor
                 * that was just closed in this iteration
                 */

                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                               "kevent: stale event %p", ev);
                continue;
            }

            if (ev->log && (ev->log->log_level & NGX_LOG_DEBUG_CONNECTION)) {
                ngx_kqueue_dump_event(ev->log, &event_list[i]);
            }

            if (ev->oneshot) {
                ev->active = 0;
            }

            ev->available = event_list[i].data;

            if (event_list[i].flags & EV_EOF) {
                ev->pending_eof = 1;
                ev->kq_errno = event_list[i].fflags;
            }

            ev->ready = 1;

            break;

        case EVFILT_VNODE:
            ev->kq_vnode = 1;

            break;

        case EVFILT_AIO:
            ev->complete = 1;
            ev->ready = 1;

            break;

#ifdef EVFILT_USER
        case EVFILT_USER:
            break;
#endif

        default:
            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                          "unexpected kevent() filter %d",
                          event_list[i].filter);
            continue;
        }
				
				...

        ev->handler(ev);
    }

    return NGX_OK;
}

• 각 event의 종류별로 udata에 들어가 있는 정보가 다르다.
• 즉, ev->handler(ev); 부분은 udata에 저장되어 있던 handler 함수가 실행되는 것이고, 이 함수는 event의 종류별로 다르다.
• 결국, http_request에 따른 server block 선택 알고리즘을 알아내기 위해서는, http_request 를 read할 경우의 event_handler 함수를 찾아보면 될 것이다.

ngx_http_request_handler()

static void
ngx_http_request_handler(ngx_event_t *ev)
{
    ngx_connection_t    *c;
    ngx_http_request_t  *r;

    c = ev->data;
    r = c->data;

    ngx_http_set_log_request(c->log, r);

    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
                   "http run request: \"%V?%V\"", &r->uri, &r->args);

    if (c->close) {
        r->main->count++;
        ngx_http_terminate_request(r, 0);
        ngx_http_run_posted_requests(c);
        return;
    }

    if (ev->delayed && ev->timedout) {
        ev->delayed = 0;
        ev->timedout = 0;
    }

    if (ev->write) {
        r->write_event_handler(r);

    } else {
        r->read_event_handler(r);
    }

    ngx_http_run_posted_requests(c);
}

• ev->write 라면 write_event_handler()를 호출한다.
• ev->read 라면 read_event_handler()를 호출한다.
• 그 후에 ngx_http_run_poseted_requests()를 호출한다.

read_event_handler() 호출한 경우

ngx_http_process_request()

void
ngx_http_process_request(ngx_http_request_t *r)
{
    ngx_connection_t  *c;

    c = r->connection;

		...

    c->read->handler = ngx_http_request_handler;
    c->write->handler = ngx_http_request_handler;
    r->read_event_handler = ngx_http_block_reading;

    ngx_http_handler(r);
}

• 어쨌든 read_event_handler()는 ngx_http_process_request() 를 호출하게 된다.

• ngx_http_handler()

  void
  ngx_http_handler(ngx_http_request_t *r)
  {
      ngx_http_core_main_conf_t  *cmcf;
  
      r->connection->log->action = NULL;
  
      if (!r->internal) {
          switch (r->headers_in.connection_type) {
          case 0:
              r->keepalive = (r->http_version > NGX_HTTP_VERSION_10);
              break;
  
          case NGX_HTTP_CONNECTION_CLOSE:
              r->keepalive = 0;
              break;
  
          case NGX_HTTP_CONNECTION_KEEP_ALIVE:
              r->keepalive = 1;
              break;
          }
  
          r->lingering_close = (r->headers_in.content_length_n > 0
                                || r->headers_in.chunked);
          r->phase_handler = 0;
  
      } else {
          cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module);
          r->phase_handler = cmcf->phase_engine.server_rewrite_index;
      }
  
      r->valid_location = 1;
  #if (NGX_HTTP_GZIP)
      r->gzip_tested = 0;
      r->gzip_ok = 0;
      r->gzip_vary = 0;
  #endif
  
      r->write_event_handler = ngx_http_core_run_phases;
      ngx_http_core_run_phases(r);
  }

• 이 함수는 ngx_http_core_run_phases()를 호출하게 된다.

중간 정리

• 결국 nginx 또한 내부적으로 kqueue를 통해 동작하므로, 패킷이 하나씩 들어올 때 파싱을 하면서 중간중간에 파싱 상태를 저장해 놓는다.
• 상태를 저장해 놓는다는 사실은, r->write_event_handler = ngx_http_core_run_phases; 이 부분을 통해서 알 수 있다.
• 해당 이벤트가 다시 발생해서 작업을 재개할 때 저장해 놓은 상태를 통해 올바른 순서로 진행한다.

다시 재개

• ngx_http_core_run_phases()

  void
  ngx_http_core_run_phases(ngx_http_request_t *r)
  {
      ngx_int_t                   rc;
      ngx_http_phase_handler_t   *ph;
      ngx_http_core_main_conf_t  *cmcf;
  
      cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module);
  
      ph = cmcf->phase_engine.handlers;
  
      while (ph[r->phase_handler].checker) {
  
          rc = ph[r->phase_handler].checker(r, &ph[r->phase_handler]);
  
          if (rc == NGX_OK) {
              return;
          }
      }
  }

• 사실 뭘 하는지는 모르겠음. 다음으로 넘어가자.

ngx_http_run_posted_requests() 로 넘어감

• ngx_http_run_posted_requests()

  void
  ngx_http_run_posted_requests(ngx_connection_t *c)
  {
      ngx_http_request_t         *r;
      ngx_http_posted_request_t  *pr;
  
      for ( ;; ) {
  
          if (c->destroyed) {
              return;
          }
  
          r = c->data;
          pr = r->main->posted_requests;
  
          if (pr == NULL) {
              return;
          }
  
          r->main->posted_requests = pr->next;
  
          r = pr->request;
  
          ngx_http_set_log_request(c->log, r);
  
          ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
                         "http posted request: \"%V?%V\"", &r->uri, &r->args);
  
          r->write_event_handler(r);
      }
  }

• 얘는 handle되지 않은 posted events들에 대해서 handler를 호출하는 함수이다.

• ngx_http_find_virtual_server()

  static ngx_int_t
  ngx_http_find_virtual_server(ngx_connection_t *c,
      ngx_http_virtual_names_t *virtual_names, ngx_str_t *host,
      ngx_http_request_t *r, ngx_http_core_srv_conf_t **cscfp)
  {
      ngx_http_core_srv_conf_t  *cscf;
  
      if (virtual_names == NULL) {
          return NGX_DECLINED;
      }
  
      cscf = ngx_hash_find_combined(&virtual_names->names,
                                    ngx_hash_key(host->data, host->len),
                                    host->data, host->len);
  
      if (cscf) {
          *cscfp = cscf;
          return NGX_OK;
      }
  
  #if (NGX_PCRE)
  
      if (host->len && virtual_names->nregex) {
          ngx_int_t                n;
          ngx_uint_t               i;
          ngx_http_server_name_t  *sn;
  
          sn = virtual_names->regex;
  
  #if (NGX_HTTP_SSL && defined SSL_CTRL_SET_TLSEXT_HOSTNAME)
  
          if (r == NULL) {
              ngx_http_connection_t  *hc;
  
              for (i = 0; i < virtual_names->nregex; i++) {
  
                  n = ngx_regex_exec(sn[i].regex->regex, host, NULL, 0);
  
                  if (n == NGX_REGEX_NO_MATCHED) {
                      continue;
                  }
  
                  if (n >= 0) {
                      hc = c->data;
                      hc->ssl_servername_regex = sn[i].regex;
  
                      *cscfp = sn[i].server;
                      return NGX_OK;
                  }
  
                  ngx_log_error(NGX_LOG_ALERT, c->log, 0,
                                ngx_regex_exec_n " failed: %i "
                                "on \"%V\" using \"%V\"",
                                n, host, &sn[i].regex->name);
  
                  return NGX_ERROR;
              }
  
              return NGX_DECLINED;
          }
  
  #endif /* NGX_HTTP_SSL && defined SSL_CTRL_SET_TLSEXT_HOSTNAME */
  
          for (i = 0; i < virtual_names->nregex; i++) {
  
              n = ngx_http_regex_exec(r, sn[i].regex, host);
  
              if (n == NGX_DECLINED) {
                  continue;
              }
  
              if (n == NGX_OK) {
                  *cscfp = sn[i].server;
                  return NGX_OK;
              }
  
              return NGX_ERROR;
          }
      }
  
  #endif /* NGX_PCRE */
  
      return NGX_DECLINED;
  }

HTTP Connection 처리 함수들

• ngx_event_accept()

  • ngx_get_connection()

    ```c
    ngx_connection_t *
    ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
    {
        ngx_uint_t         instance;
        ngx_event_t       *rev, *wev;
        ngx_connection_t  *c;
    
        /* disable warning: Win32 SOCKET is u_int while UNIX socket is int */
    
    		...
    
        ngx_drain_connections((ngx_cycle_t *) ngx_cycle);
    
        c = ngx_cycle->free_connections;
    
    		...
    
        ngx_cycle->free_connections = c->data;
        ngx_cycle->free_connection_n--;
    
        if (ngx_cycle->files && ngx_cycle->files[s] == NULL) {
            ngx_cycle->files[s] = c;
        }
    
        rev = c->read;
        wev = c->write;
    
        ngx_memzero(c, sizeof(ngx_connection_t));
    
        c->read = rev;
        c->write = wev;
        c->fd = s;
        c->log = log;
    
        instance = rev->instance;
    
        ngx_memzero(rev, sizeof(ngx_event_t));
        ngx_memzero(wev, sizeof(ngx_event_t));
    
        rev->instance = !instance;
        wev->instance = !instance;
    
        rev->index = NGX_INVALID_INDEX;
        wev->index = NGX_INVALID_INDEX;
    
        rev->data = c;
        wev->data = c;
    
        wev->write = 1;
    
        return c;
    }
    ```
    
    - 그냥 별 거 없는듯.
    - 단순히 free 상태의 connection 객체를 초기화해서 리턴해 주는 함수이다.

    void
    ngx_event_accept(ngx_event_t *ev)
    {
        socklen_t          socklen;
        ngx_err_t          err;
        ngx_log_t         *log;
        ngx_uint_t         level;
        ngx_socket_t       s;
        ngx_event_t       *rev, *wev;
        ngx_sockaddr_t     sa;
        ngx_listening_t   *ls;
        ngx_connection_t  *c, *lc;
        ngx_event_conf_t  *ecf;
    
    		...
    
        ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
    
        if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) {
            ev->available = ecf->multi_accept;
        }
    
        lc = ev->data;
        ls = lc->listening;
        ev->ready = 0;
    
    		...
    
        do {
            socklen = sizeof(ngx_sockaddr_t);
            s = accept(lc->fd, &sa.sockaddr, &socklen);
    
    				...
    
            ngx_accept_disabled = ngx_cycle->connection_n / 8
                                  - ngx_cycle->free_connection_n;
    
            c = ngx_get_connection(s, ev->log);
    
    				...
    
            c->type = SOCK_STREAM;
            c->pool = ngx_create_pool(ls->pool_size, ev->log);
    
    				...
    
            c->sockaddr = ngx_palloc(c->pool, socklen);
    
    				...
    
            ngx_memcpy(c->sockaddr, &sa, socklen);
    
    				...
    
            c->recv = ngx_recv;
            c->send = ngx_send;
            c->recv_chain = ngx_recv_chain;
            c->send_chain = ngx_send_chain;
    
            c->log = log;
            c->pool->log = log;
    
            c->socklen = socklen;
            c->listening = ls;
            c->local_sockaddr = ls->sockaddr;
            c->local_socklen = ls->socklen;
    
            if (c->sockaddr->sa_family == AF_UNIX) {
                c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
                c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
            }
    
            rev = c->read;
            wev = c->write;
    
            wev->ready = 1;
    
            if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
                rev->ready = 1;
            }
    
            if (ev->deferred_accept) {
                rev->ready = 1;
                rev->available = 1;
            }
    
    				...
    
            /*
             * TODO: MT: - ngx_atomic_fetch_add()
             *             or protection by critical section or light mutex
             *
             * TODO: MP: - allocated in a shared memory
             *           - ngx_atomic_fetch_add()
             *             or protection by critical section or light mutex
             */
    
            c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
    
            c->start_time = ngx_current_msec;
    
            if (ls->addr_ntop) {
                c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
                if (c->addr_text.data == NULL) {
                    ngx_close_accepted_connection(c);
                    return;
                }
    
                c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
                                                 c->addr_text.data,
                                                 ls->addr_text_max_len, 0);
                if (c->addr_text.len == 0) {
                    ngx_close_accepted_connection(c);
                    return;
                }
            }
    
            if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) {
                if (ngx_add_conn(c) == NGX_ERROR) {
                    ngx_close_accepted_connection(c);
                    return;
                }
            }
    
    				...
    
            ls->handler(c); // 여기서 ngx_http_init_connection()이 호출됨.
    
            if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
                ev->available--;
            }
    
        } while (ev->available);
    
    #if (NGX_HAVE_EPOLLEXCLUSIVE)
        ngx_reorder_accept_events(ls);
    #endif
    }

• ngx_http_init_connection()

• ngx_connection_local_sockaddr()

    ngx_int_t
  ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
      ngx_uint_t port)
  {
      socklen_t             len;
      ngx_uint_t            addr;
      ngx_sockaddr_t        sa;
      struct sockaddr_in   *sin;
  
      addr = 0;
  
      if (c->local_socklen) {
          switch (c->local_sockaddr->sa_family) {
          default: /* AF_INET */
              sin = (struct sockaddr_in *) c->local_sockaddr;
              addr = sin->sin_addr.s_addr;
              break;
          }
      }
  
      if (addr == 0) {
  
          len = sizeof(ngx_sockaddr_t);
  
          if (getsockname(c->fd, &sa.sockaddr, &len) == -1) {
              ngx_connection_error(c, ngx_socket_errno, "getsockname() failed");
              return NGX_ERROR;
          }
  
          c->local_sockaddr = ngx_palloc(c->pool, len);
          if (c->local_sockaddr == NULL) {
              return NGX_ERROR;
          }
  
          ngx_memcpy(c->local_sockaddr, &sa, len);
  
          c->local_socklen = len;
      }
  
      if (s == NULL) {
          return NGX_OK;
      }
  
      s->len = ngx_sock_ntop(c->local_sockaddr, c->local_socklen,
                             s->data, s->len, port);
  
      return NGX_OK;
  }