corosync  3.0.3
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  qb_loop_t *poll_handle;
97 
98  knet_handle_t knet_handle;
99 
101 
102  void *context;
103 
105  void *context,
106  const void *msg,
107  unsigned int msg_len,
108  const struct sockaddr_storage *system_from);
109 
111  void *context,
112  const struct totem_ip_address *iface_address,
113  unsigned int link_no);
114 
116  void *context,
117  int net_mtu);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137 
139  int level,
140  int subsys,
141  const char *function,
142  const char *file,
143  int line,
144  const char *format,
145  ...)__attribute__((format(printf, 6, 7)));
146 
147  void *knet_context;
148 
149  char iov_buffer[KNET_MAX_PACKET_SIZE];
150 
152 
154 
156 
158 
160 
162 
164 
165  qb_loop_timer_handle timer_netif_check_timeout;
166 
167  qb_loop_timer_handle timer_merge_detect_timeout;
168 
170 
172 
173  int logpipes[2];
174  int knet_fd;
175 
176  pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178  char *nozzle_name;
182  nozzle_t nozzle_handle;
183 #endif
184 };
185 
186 /* Awkward. But needed to get stats from knet */
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
196  void *knet_context);
197 
198 static void totemknet_start_merge_detect_timeout(
199  void *knet_context);
200 
201 static void totemknet_stop_merge_detect_timeout(
202  void *knet_context);
203 
204 static void log_flush_messages (
205  void *knet_context);
206 
207 static void totemknet_instance_initialize (struct totemknet_instance *instance)
208 {
209  int res;
210 
211  memset (instance, 0, sizeof (struct totemknet_instance));
212  res = pthread_mutex_init(&instance->log_mutex, NULL);
213  /*
214  * There is not too much else what can be done.
215  */
216  assert(res == 0);
217 }
218 
219 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
220 do { \
221  (void)pthread_mutex_lock(&instance->log_mutex); \
222  instance->totemknet_log_printf ( \
223  level, subsys, function, file, line, \
224  (const char *)format, ##args); \
225  (void)pthread_mutex_unlock(&instance->log_mutex); \
226 } while (0);
227 
228 #define knet_log_printf(level, format, args...) \
229 do { \
230  knet_log_printf_lock ( \
231  level, instance->totemknet_subsys_id, \
232  __FUNCTION__, __FILE__, __LINE__, \
233  (const char *)format, ##args); \
234 } while (0);
235 
236 #define libknet_log_printf(level, format, args...) \
237 do { \
238  knet_log_printf_lock ( \
239  level, instance->knet_subsys_id, \
240  __FUNCTION__, "libknet.h", __LINE__, \
241  (const char *)format, ##args); \
242 } while (0);
243 
244 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
245 do { \
246  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
247  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
248  instance->totemknet_log_printf ( \
249  level, instance->totemknet_subsys_id, \
250  __FUNCTION__, __FILE__, __LINE__, \
251  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
252  } while(0)
253 
254 
255 #ifdef HAVE_LIBNOZZLE
256 static inline int is_ether_addr_multicast(const uint8_t *addr)
257 {
258  return (addr[0] & 0x01);
259 }
260 static inline int is_ether_addr_zero(const uint8_t *addr)
261 {
262  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
263 }
264 
265 static int ether_host_filter_fn(void *private_data,
266  const unsigned char *outdata,
267  ssize_t outdata_len,
268  uint8_t tx_rx,
269  knet_node_id_t this_host_id,
270  knet_node_id_t src_host_id,
271  int8_t *channel,
272  knet_node_id_t *dst_host_ids,
273  size_t *dst_host_ids_entries)
274 {
275  struct ether_header *eth_h = (struct ether_header *)outdata;
276  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
277  uint16_t dst_host_id;
278 
279  if (is_ether_addr_zero(dst_mac))
280  return -1;
281 
282  if (is_ether_addr_multicast(dst_mac)) {
283  return 1;
284  }
285 
286  memmove(&dst_host_id, &dst_mac[4], 2);
287 
288  dst_host_ids[0] = ntohs(dst_host_id);
289  *dst_host_ids_entries = 1;
290 
291  return 0;
292 }
293 #endif
294 
295 static int dst_host_filter_callback_fn(void *private_data,
296  const unsigned char *outdata,
297  ssize_t outdata_len,
298  uint8_t tx_rx,
299  knet_node_id_t this_host_id,
300  knet_node_id_t src_host_id,
301  int8_t *channel,
302  knet_node_id_t *dst_host_ids,
303  size_t *dst_host_ids_entries)
304 {
305  struct totem_message_header *header = (struct totem_message_header *)outdata;
306  int res;
307 
308 #ifdef HAVE_LIBNOZZLE
309  if (*channel != 0) {
310  return ether_host_filter_fn(private_data,
311  outdata, outdata_len,
312  tx_rx,
313  this_host_id, src_host_id,
314  channel,
315  dst_host_ids,
316  dst_host_ids_entries);
317  }
318 #endif
319  if (header->target_nodeid) {
320  dst_host_ids[0] = header->target_nodeid;
321  *dst_host_ids_entries = 1;
322  res = 0; /* unicast message */
323  }
324  else {
325  *dst_host_ids_entries = 0;
326  res = 1; /* multicast message */
327  }
328  return res;
329 }
330 
331 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
332 {
333  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
334 
335  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
336  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
337  knet_handle_remove_datafd(instance->knet_handle, datafd);
338  }
339 }
340 
341 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
342 {
343  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
344 
345  // TODO: what? if anything.
346  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
347 }
348 
349 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
350 {
351  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
352  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
353 
354  /* We don't need to tell corosync the actual knet MTU */
355 // instance->totemknet_mtu_changed(instance->context, data_mtu);
356 }
357 
359  void *knet_context,
360  const char *cipher_type,
361  const char *hash_type)
362 {
363  return (0);
364 }
365 
366 
367 static inline void ucast_sendmsg (
368  struct totemknet_instance *instance,
369  struct totem_ip_address *system_to,
370  const void *msg,
371  unsigned int msg_len)
372 {
373  int res = 0;
374  struct totem_message_header *header = (struct totem_message_header *)msg;
375  struct msghdr msg_ucast;
376  struct iovec iovec;
377 
378  header->target_nodeid = system_to->nodeid;
379 
380  iovec.iov_base = (void *)msg;
381  iovec.iov_len = msg_len;
382 
383  /*
384  * Build unicast message
385  */
386  memset(&msg_ucast, 0, sizeof(msg_ucast));
387  msg_ucast.msg_iov = (void *)&iovec;
388  msg_ucast.msg_iovlen = 1;
389 #ifdef HAVE_MSGHDR_CONTROL
390  msg_ucast.msg_control = 0;
391 #endif
392 #ifdef HAVE_MSGHDR_CONTROLLEN
393  msg_ucast.msg_controllen = 0;
394 #endif
395 #ifdef HAVE_MSGHDR_FLAGS
396  msg_ucast.msg_flags = 0;
397 #endif
398 #ifdef HAVE_MSGHDR_ACCRIGHTS
399  msg_ucast.msg_accrights = NULL;
400 #endif
401 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
402  msg_ucast.msg_accrightslen = 0;
403 #endif
404 
405  /*
406  * Transmit unicast message
407  * An error here is recovered by totemsrp
408  */
409 
410  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
411  if (res < 0) {
413  "sendmsg(ucast) failed (non-critical)");
414  }
415 }
416 
417 static inline void mcast_sendmsg (
418  struct totemknet_instance *instance,
419  const void *msg,
420  unsigned int msg_len,
421  int only_active)
422 {
423  int res;
424  struct totem_message_header *header = (struct totem_message_header *)msg;
425  struct msghdr msg_mcast;
426  struct iovec iovec;
427 
428  iovec.iov_base = (void *)msg;
429  iovec.iov_len = msg_len;
430 
431  header->target_nodeid = 0;
432 
433  /*
434  * Build multicast message
435  */
436  memset(&msg_mcast, 0, sizeof(msg_mcast));
437  msg_mcast.msg_iov = (void *)&iovec;
438  msg_mcast.msg_iovlen = 1;
439 #ifdef HAVE_MSGHDR_CONTROL
440  msg_mcast.msg_control = 0;
441 #endif
442 #ifdef HAVE_MSGHDR_CONTROLLEN
443  msg_mcast.msg_controllen = 0;
444 #endif
445 #ifdef HAVE_MSGHDR_FLAGS
446  msg_mcast.msg_flags = 0;
447 #endif
448 #ifdef HAVE_MSGHDR_ACCRIGHTS
449  msg_mcast.msg_accrights = NULL;
450 #endif
451 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
452  msg_mcast.msg_accrightslen = 0;
453 #endif
454 
455 
456 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
457 
458  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
459  if (res < msg_len) {
460  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
461  }
462 
463  if (!only_active || instance->send_merge_detect_message) {
464  /*
465  * Current message was sent to all nodes
466  */
468  instance->send_merge_detect_message = 0;
469  }
470 }
471 
472 static int node_compare(const void *aptr, const void *bptr)
473 {
474  uint16_t a,b;
475 
476  a = *(uint16_t *)aptr;
477  b = *(uint16_t *)bptr;
478 
479  return a > b;
480 }
481 
482 int totemknet_ifaces_get (void *knet_context,
483  char ***status,
484  unsigned int *iface_count)
485 {
486  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
487  struct knet_link_status link_status;
488  knet_node_id_t host_list[KNET_MAX_HOST];
489  uint8_t link_list[KNET_MAX_LINK];
490  size_t num_hosts;
491  size_t num_links;
492  size_t link_idx;
493  int i,j;
494  char *ptr;
495  int res = 0;
496 
497  /*
498  * Don't do the whole 'link_info' bit if the caller just wants
499  * a count of interfaces.
500  */
501  if (status) {
502 
503  res = knet_host_get_host_list(instance->knet_handle,
504  host_list, &num_hosts);
505  if (res) {
506  return (-1);
507  }
508  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
509 
510  for (i=0; i<INTERFACE_MAX; i++) {
511  memset(instance->link_status[i], 'n', CFG_INTERFACE_STATUS_MAX_LEN-1);
512  instance->link_status[i][num_hosts] = '\0';
513  }
514 
515  /* This is all a bit "inside-out" because "status" is a set of strings per link
516  * and knet orders things by host
517  */
518  for (j=0; j<num_hosts; j++) {
519  res = knet_link_get_link_list(instance->knet_handle,
520  host_list[j], link_list, &num_links);
521  if (res) {
522  return (-1);
523  }
524 
525  link_idx = 0;
526  for (i=0; i < num_links; i++) {
527  /*
528  * Skip over links that are unconfigured to corosync. This is basically
529  * link0 if corosync isn't using it for comms, as we will still
530  * have it set up for loopback.
531  */
532  if (!instance->totem_config->interfaces[link_list[i]].configured) {
533  continue;
534  }
535  ptr = instance->link_status[link_idx++];
536 
537  res = knet_link_get_status(instance->knet_handle,
538  host_list[j],
539  link_list[i],
540  &link_status,
541  sizeof(link_status));
542  if (res == 0) {
543  ptr[j] = '0' + (link_status.enabled |
544  link_status.connected<<1 |
545  link_status.dynconnected<<2);
546  }
547  else {
548  ptr[j] = '?';
549  }
550  }
551  }
552  *status = instance->link_status;
553  }
554 
555  *iface_count = INTERFACE_MAX;
556 
557  return (res);
558 }
559 
561  void *knet_context)
562 {
563  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
564  int res = 0;
565  int i,j;
566  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
567  uint8_t links[KNET_MAX_LINK];
568  size_t num_nodes;
569  size_t num_links;
570 
571  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
572 
573  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
574  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
575 
576  /*
577  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
578  */
579  res = knet_handle_setfwd(instance->knet_handle, 0);
580  if (res) {
581  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
582  }
583 
584  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
585  if (res) {
586  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
587  /* Crash out anyway */
588  goto finalise_error;
589  }
590 
591  /* Tidily shut down all nodes & links. */
592  for (i=0; i<num_nodes; i++) {
593 
594  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
595  if (res) {
596  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
597  goto finalise_error;
598  }
599  for (j=0; j<num_links; j++) {
600  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
601  if (res) {
602  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
603  }
604  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
605  if (res) {
606  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
607  }
608  }
609  res = knet_host_remove(instance->knet_handle, nodes[i]);
610  if (res) {
611  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
612  }
613  }
614 
615 finalise_error:
616  res = knet_handle_free(instance->knet_handle);
617  if (res) {
618  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
619  }
620 
621  totemknet_stop_merge_detect_timeout(instance);
622 
623  log_flush_messages(instance);
624 
625  /*
626  * Error is deliberately ignored
627  */
628  (void)pthread_mutex_destroy(&instance->log_mutex);
629 
630  return (res);
631 }
632 
633 static int log_deliver_fn (
634  int fd,
635  int revents,
636  void *data)
637 {
638  struct totemknet_instance *instance = (struct totemknet_instance *)data;
639  char buffer[sizeof(struct knet_log_msg)*4];
640  char *bufptr = buffer;
641  int done = 0;
642  int len;
643 
644  len = read(fd, buffer, sizeof(buffer));
645  while (done < len) {
646  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
647  switch (msg->msglevel) {
648  case KNET_LOG_ERR:
650  knet_log_get_subsystem_name(msg->subsystem),
651  msg->msg);
652  break;
653  case KNET_LOG_WARN:
655  knet_log_get_subsystem_name(msg->subsystem),
656  msg->msg);
657  break;
658  case KNET_LOG_INFO:
660  knet_log_get_subsystem_name(msg->subsystem),
661  msg->msg);
662  break;
663  case KNET_LOG_DEBUG:
665  knet_log_get_subsystem_name(msg->subsystem),
666  msg->msg);
667  break;
668  }
669  bufptr += sizeof(struct knet_log_msg);
670  done += sizeof(struct knet_log_msg);
671  }
672  return 0;
673 }
674 
675 static int data_deliver_fn (
676  int fd,
677  int revents,
678  void *data)
679 {
680  struct totemknet_instance *instance = (struct totemknet_instance *)data;
681  struct msghdr msg_hdr;
682  struct iovec iov_recv;
683  struct sockaddr_storage system_from;
684  ssize_t msg_len;
685  int truncated_packet;
686 
687  iov_recv.iov_base = instance->iov_buffer;
688  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
689 
690  msg_hdr.msg_name = &system_from;
691  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
692  msg_hdr.msg_iov = &iov_recv;
693  msg_hdr.msg_iovlen = 1;
694 #ifdef HAVE_MSGHDR_CONTROL
695  msg_hdr.msg_control = 0;
696 #endif
697 #ifdef HAVE_MSGHDR_CONTROLLEN
698  msg_hdr.msg_controllen = 0;
699 #endif
700 #ifdef HAVE_MSGHDR_FLAGS
701  msg_hdr.msg_flags = 0;
702 #endif
703 #ifdef HAVE_MSGHDR_ACCRIGHTS
704  msg_hdr.msg_accrights = NULL;
705 #endif
706 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
707  msg_hdr.msg_accrightslen = 0;
708 #endif
709 
710  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
711  if (msg_len <= 0) {
712  return (0);
713  }
714 
715  truncated_packet = 0;
716 
717 #ifdef HAVE_MSGHDR_FLAGS
718  if (msg_hdr.msg_flags & MSG_TRUNC) {
719  truncated_packet = 1;
720  }
721 #else
722  /*
723  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
724  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
725  */
726  if (bytes_received == KNET_MAX_PACKET_SIZE) {
727  truncated_packet = 1;
728  }
729 #endif
730 
731  if (truncated_packet) {
733  "Received too big message. This may be because something bad is happening"
734  "on the network (attack?), or you tried join more nodes than corosync is"
735  "compiled with (%u) or bug in the code (bad estimation of "
736  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
737  return (0);
738  }
739 
740  /*
741  * Handle incoming message
742  */
743  instance->totemknet_deliver_fn (
744  instance->context,
745  instance->iov_buffer,
746  msg_len,
747  &system_from);
748 
749  return (0);
750 }
751 
752 static void timer_function_netif_check_timeout (
753  void *data)
754 {
755  struct totemknet_instance *instance = (struct totemknet_instance *)data;
756  int i;
757 
758  for (i=0; i < INTERFACE_MAX; i++) {
759  if (!instance->totem_config->interfaces[i].configured) {
760  continue;
761  }
762  instance->totemknet_iface_change_fn (instance->context,
763  &instance->my_ids[i],
764  i);
765  }
766 }
767 
768 static void knet_set_access_list_config(struct totemknet_instance *instance)
769 {
770 #ifdef HAVE_KNET_ACCESS_LIST
771  uint32_t value;
772  cs_error_t err;
773 
775  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
776 
777  err = knet_handle_enable_access_lists(instance->knet_handle, value);
778  if (err) {
779  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
780  }
781 #endif
782 }
783 
784 
785 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
786 static void totemknet_refresh_config(
787  int32_t event,
788  const char *key_name,
789  struct icmap_notify_value new_val,
790  struct icmap_notify_value old_val,
791  void *user_data)
792 {
793  uint8_t reloading;
794  uint32_t value;
795  uint32_t link_no;
796  size_t num_nodes;
797  knet_node_id_t host_ids[KNET_MAX_HOST];
798  int i;
799  int err;
800  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
801 
802  ENTER();
803 
804  /*
805  * If a full reload is in progress then don't do anything until it's done and
806  * can reconfigure it all atomically
807  */
808  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
809  return;
810  }
811 
812  knet_set_access_list_config(instance);
813 
814  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
815 
817  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
818  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
819  if (err) {
820  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
821  }
822  }
823 
824  /* Configure link parameters for each node */
825  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
826  if (err != 0) {
827  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
828  }
829 
830  for (i=0; i<num_nodes; i++) {
831  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
832  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
833  continue;
834  }
835 
836  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
837  instance->totem_config->interfaces[link_no].knet_ping_interval,
838  instance->totem_config->interfaces[link_no].knet_ping_timeout,
839  instance->totem_config->interfaces[link_no].knet_ping_precision);
840  if (err) {
841  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
842  }
843  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
844  instance->totem_config->interfaces[link_no].knet_pong_count);
845  if (err) {
846  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
847  }
848  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
849  instance->totem_config->interfaces[link_no].knet_link_priority);
850  if (err) {
851  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
852  }
853 
854  }
855  }
856 
857  LEAVE();
858 }
859 
860 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
861 {
862  icmap_track_t icmap_track_totem = NULL;
863  icmap_track_t icmap_track_reload = NULL;
864 
865  ENTER();
866 
867  icmap_track_add("totem.",
869  totemknet_refresh_config,
870  instance,
871  &icmap_track_totem);
872 
873  icmap_track_add("config.totemconfig_reload_in_progress",
875  totemknet_refresh_config,
876  instance,
877  &icmap_track_reload);
878 
879  LEAVE();
880 }
881 
882 /*
883  * Create an instance
884  */
886  qb_loop_t *poll_handle,
887  void **knet_context,
888  struct totem_config *totem_config,
889  totemsrp_stats_t *stats,
890  void *context,
891 
892  void (*deliver_fn) (
893  void *context,
894  const void *msg,
895  unsigned int msg_len,
896  const struct sockaddr_storage *system_from),
897 
898  void (*iface_change_fn) (
899  void *context,
900  const struct totem_ip_address *iface_address,
901  unsigned int link_no),
902 
903  void (*mtu_changed) (
904  void *context,
905  int net_mtu),
906 
907  void (*target_set_completed) (
908  void *context))
909 {
910  struct totemknet_instance *instance;
911  int8_t channel=0;
912  int res;
913  int i;
914 
915  instance = malloc (sizeof (struct totemknet_instance));
916  if (instance == NULL) {
917  return (-1);
918  }
919 
920  totemknet_instance_initialize (instance);
921 
922  instance->totem_config = totem_config;
923 
924  /*
925  * Configure logging
926  */
927  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
934 
935  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
936 
937  /*
938  * Initialize local variables for totemknet
939  */
940 
941  instance->our_nodeid = instance->totem_config->node_id;
942 
943  for (i=0; i< INTERFACE_MAX; i++) {
944  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
945  instance->my_ids[i].nodeid = instance->our_nodeid;
946  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
947 
948  /* Needed for totemsrp */
950  }
951 
952  instance->poll_handle = poll_handle;
953 
954  instance->context = context;
955  instance->totemknet_deliver_fn = deliver_fn;
956 
957  instance->totemknet_iface_change_fn = iface_change_fn;
958 
959  instance->totemknet_mtu_changed = mtu_changed;
960 
961  instance->totemknet_target_set_completed = target_set_completed;
962 
963  instance->loopback_link = 0;
964 
965  res = pipe(instance->logpipes);
966  if (res == -1) {
967  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
968  goto exit_error;
969  }
970  if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
971  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
972  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
973  goto exit_error;
974  }
975 
976 
977 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
978  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
979 #else
980  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
981 #endif
982 
983  if (!instance->knet_handle && errno == ENAMETOOLONG) {
984  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
985 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
986  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
987 #else
988  instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
989 #endif
990  }
991 
992  if (!instance->knet_handle) {
993  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
994  goto exit_error;
995  }
996 
997  knet_set_access_list_config(instance);
998 
999  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1000  if (res) {
1001  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1002  }
1003  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1004  if (res) {
1005  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1006  }
1007  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1008  if (res) {
1009  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1010  }
1011  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1012  if (res) {
1013  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1014  }
1015  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1016  if (res) {
1017  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1018  }
1019  global_instance = instance;
1020 
1021  /* Get an fd into knet */
1022  instance->knet_fd = 0;
1023  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1024  if (res) {
1025  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1026  goto exit_error;
1027  }
1028 
1029  /* Enable crypto if requested */
1030  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1031  struct knet_handle_crypto_cfg crypto_cfg;
1032 
1033  assert(strlen(instance->totem_config->crypto_model) < sizeof(crypto_cfg.crypto_model));
1034  strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
1035 
1036  assert(strlen(instance->totem_config->crypto_cipher_type) < sizeof(crypto_cfg.crypto_cipher_type));
1037  strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
1038 
1039  assert(strlen(instance->totem_config->crypto_hash_type) < sizeof(crypto_cfg.crypto_hash_type));
1040  strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
1041 
1042  assert(instance->totem_config->private_key_len <= sizeof(crypto_cfg.private_key));
1043  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1044  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1045 
1046  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1047  if (res == -1) {
1048  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1049  goto exit_error;
1050  }
1051  if (res == -2) {
1052  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1053  goto exit_error;
1054  }
1055  knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
1056  }
1057 
1058  /* Set up compression */
1059  totemknet_reconfigure(instance, instance->totem_config);
1060 
1061  knet_handle_setfwd(instance->knet_handle, 1);
1062 
1063  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1064  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1065  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1066  }
1067  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1068  instance->link_mode = KNET_LINK_POLICY_RR;
1069  }
1070 
1071  for (i=0; i<INTERFACE_MAX; i++) {
1072  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1073  if (!instance->link_status[i]) {
1074  goto exit_error;
1075  }
1076  }
1077 
1078  qb_loop_poll_add (instance->poll_handle,
1079  QB_LOOP_MED,
1080  instance->logpipes[0],
1081  POLLIN, instance, log_deliver_fn);
1082 
1083  qb_loop_poll_add (instance->poll_handle,
1084  QB_LOOP_HIGH,
1085  instance->knet_fd,
1086  POLLIN, instance, data_deliver_fn);
1087 
1088  /*
1089  * Upper layer isn't ready to receive message because it hasn't
1090  * initialized yet. Add short timer to check the interfaces.
1091  */
1092  qb_loop_timer_add (instance->poll_handle,
1093  QB_LOOP_MED,
1094  100*QB_TIME_NS_IN_MSEC,
1095  (void *)instance,
1096  timer_function_netif_check_timeout,
1097  &instance->timer_netif_check_timeout);
1098 
1099  totemknet_start_merge_detect_timeout(instance);
1100 
1101  /* Start listening for config changes */
1102  totemknet_add_config_notifications(instance);
1103 
1104  /* Add stats keys to icmap */
1106 
1107  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1108  *knet_context = instance;
1109 
1110  return (0);
1111 
1112 exit_error:
1113  log_flush_messages(instance);
1114  free(instance);
1115  return (-1);
1116 }
1117 
1119 {
1120  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1121  return malloc(KNET_MAX_PACKET_SIZE + 512);
1122 }
1123 
1125 {
1126  return free (ptr);
1127 }
1128 
1130  void *knet_context,
1131  int processor_count)
1132 {
1133  return (0);
1134 }
1135 
1136 int totemknet_recv_flush (void *knet_context)
1137 {
1138  return (0);
1139 }
1140 
1141 int totemknet_send_flush (void *knet_context)
1142 {
1143  return (0);
1144 }
1145 
1147  void *knet_context,
1148  const void *msg,
1149  unsigned int msg_len)
1150 {
1151  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1152  int res = 0;
1153 
1154  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1155 
1156  return (res);
1157 }
1159  void *knet_context,
1160  const void *msg,
1161  unsigned int msg_len)
1162 {
1163  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1164  int res = 0;
1165 
1166  mcast_sendmsg (instance, msg, msg_len, 0);
1167 
1168  return (res);
1169 }
1170 
1172  void *knet_context,
1173  const void *msg,
1174  unsigned int msg_len)
1175 {
1176  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1177  int res = 0;
1178 
1179  mcast_sendmsg (instance, msg, msg_len, 1);
1180 
1181  return (res);
1182 }
1183 
1184 
1186 {
1187  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1188  int res = 0;
1189 
1190  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1191 
1192  return (res);
1193 }
1194 
1196 {
1197  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1198 
1199  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1200 }
1201 
1203  void *knet_context,
1204  unsigned int nodeid)
1205 {
1206  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1207  int res = 0;
1208 
1209  instance->token_target.nodeid = nodeid;
1210 
1211  instance->totemknet_target_set_completed (instance->context);
1212 
1213  return (res);
1214 }
1215 
1217  void *knet_context)
1218 {
1219  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1220  unsigned int res;
1221  struct sockaddr_storage system_from;
1222  struct msghdr msg_hdr;
1223  struct iovec iov_recv;
1224  struct pollfd ufd;
1225  int nfds;
1226  int msg_processed = 0;
1227 
1228  iov_recv.iov_base = instance->iov_buffer;
1229  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1230 
1231  msg_hdr.msg_name = &system_from;
1232  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1233  msg_hdr.msg_iov = &iov_recv;
1234  msg_hdr.msg_iovlen = 1;
1235 #ifdef HAVE_MSGHDR_CONTROL
1236  msg_hdr.msg_control = 0;
1237 #endif
1238 #ifdef HAVE_MSGHDR_CONTROLLEN
1239  msg_hdr.msg_controllen = 0;
1240 #endif
1241 #ifdef HAVE_MSGHDR_FLAGS
1242  msg_hdr.msg_flags = 0;
1243 #endif
1244 #ifdef HAVE_MSGHDR_ACCRIGHTS
1245  msg_msg_hdr.msg_accrights = NULL;
1246 #endif
1247 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1248  msg_msg_hdr.msg_accrightslen = 0;
1249 #endif
1250 
1251  do {
1252  ufd.fd = instance->knet_fd;
1253  ufd.events = POLLIN;
1254  nfds = poll (&ufd, 1, 0);
1255  if (nfds == 1 && ufd.revents & POLLIN) {
1256  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1257  if (res != -1) {
1258  msg_processed = 1;
1259  } else {
1260  msg_processed = -1;
1261  }
1262  }
1263  } while (nfds == 1);
1264 
1265  return (msg_processed);
1266 }
1267 
1268 int totemknet_iface_set (void *knet_context,
1269  const struct totem_ip_address *local_addr,
1270  unsigned short ip_port,
1271  unsigned int iface_no)
1272 {
1273  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1274 
1275  totemip_copy(&instance->my_ids[iface_no], local_addr);
1276 
1277  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1278 
1279  instance->ip_port[iface_no] = ip_port;
1280 
1281  return 0;
1282 }
1283 
1284 
1286  void *knet_context,
1287  const struct totem_ip_address *local,
1288  const struct totem_ip_address *member,
1289  int link_no)
1290 {
1291  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1292  int err;
1293  int port = instance->ip_port[link_no];
1294  struct sockaddr_storage remote_ss;
1295  struct sockaddr_storage local_ss;
1296  int addrlen;
1297  int i;
1298  int host_found = 0;
1299  knet_node_id_t host_ids[KNET_MAX_HOST];
1300  size_t num_host_ids;
1301 
1302  /* Only create 1 loopback link and use link 0 */
1303  if (member->nodeid == instance->our_nodeid) {
1304  if (!instance->loopback_link) {
1305  link_no = 0;
1306  instance->loopback_link = 1;
1307  } else {
1308  /* Already done */
1309  return 0;
1310  }
1311  }
1312 
1313  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1314  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1315 
1316 
1317  /* Only add the host if it doesn't already exist in knet */
1318  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1319  if (err) {
1320  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1321  return -1;
1322  }
1323  for (i=0; i<num_host_ids; i++) {
1324  if (host_ids[i] == member->nodeid) {
1325  host_found = 1;
1326  }
1327  }
1328 
1329  if (!host_found) {
1330  err = knet_host_add(instance->knet_handle, member->nodeid);
1331  if (err != 0 && errno != EEXIST) {
1332  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1333  return -1;
1334  }
1335  } else {
1336  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1337  }
1338 
1339 
1340  if (err == 0) {
1341  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1342  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1343  return -1;
1344  }
1345  }
1346 
1347  memset(&local_ss, 0, sizeof(local_ss));
1348  memset(&remote_ss, 0, sizeof(remote_ss));
1349  /* Casts to remove const */
1350  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1351  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1352 
1353  if (member->nodeid == instance->our_nodeid) {
1354  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1355 
1356  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1357  KNET_TRANSPORT_LOOPBACK,
1358  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1359  }
1360  else {
1361  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1362  instance->totem_config->interfaces[link_no].knet_transport,
1363  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1364  }
1365  if (err) {
1366  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1367  return -1;
1368  }
1369 
1370  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1371  instance->totem_config->interfaces[link_no].knet_link_priority);
1372 
1373  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1374  instance->totem_config->interfaces[link_no].knet_link_priority);
1375  if (err) {
1376  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1377  }
1378 
1379  /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1380  get done in totemknet_refresh_config */
1381  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1382  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1383  instance->totem_config->interfaces[link_no].knet_ping_interval,
1384  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1385  instance->totem_config->interfaces[link_no].knet_ping_precision);
1386  if (err) {
1387  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1388  }
1389  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1390  instance->totem_config->interfaces[link_no].knet_pong_count);
1391  if (err) {
1392  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1393  }
1394  }
1395 
1396  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1397  if (err) {
1398  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1399  return -1;
1400  }
1401 
1402  /* register stats */
1403  stats_knet_add_member(member->nodeid, link_no);
1404  return (0);
1405 }
1406 
1408  void *knet_context,
1409  const struct totem_ip_address *token_target,
1410  int link_no)
1411 {
1412  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1413  int res;
1414  uint8_t link_list[KNET_MAX_LINK];
1415  size_t num_links;
1416 
1417  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1418 
1419  /* Don't remove the link with the loopback on it until we shut down */
1420  if (token_target->nodeid == instance->our_nodeid) {
1421  return 0;
1422  }
1423 
1424  /* Tidy stats */
1426 
1427  /* Remove the link first */
1428  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1429  if (res != 0) {
1430  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1431  return res;
1432  }
1433 
1434  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1435  if (res != 0) {
1436  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1437  return res;
1438  }
1439 
1440  /* If this is the last link, then remove the node */
1441  res = knet_link_get_link_list(instance->knet_handle,
1442  token_target->nodeid, link_list, &num_links);
1443  if (res) {
1444  return (0); /* not really failure */
1445  }
1446 
1447  if (num_links == 0) {
1448  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1449  }
1450  return res;
1451 }
1452 
1454  void *knet_context)
1455 {
1456  return (0);
1457 }
1458 
1460  void *knet_context,
1461  struct totem_config *totem_config)
1462 {
1463  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1464  struct knet_handle_compress_cfg compress_cfg;
1465  int res = 0;
1466 
1468  assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1469  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1470 
1471  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1472  compress_cfg.compress_level = totem_config->knet_compression_level;
1473 
1474  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1475  if (res) {
1476  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1477  }
1478  }
1479 
1480 #ifdef HAVE_LIBNOZZLE
1481  /* Set up nozzle device(s). Return code is ignored, because unability
1482  * configure nozzle is not fatal problem, errors are logged and
1483  * there is not much else we can do */
1484  (void)setup_nozzle(instance);
1485 #endif
1486  return (res);
1487 }
1488 
1489 
1491  void *knet_context)
1492 {
1493  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1494 
1495  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1496 }
1497 
1498 /* For the stats module */
1500  knet_node_id_t node, uint8_t link_no,
1501  struct knet_link_status *status)
1502 {
1503  int res;
1504  int ret = CS_OK;
1505 
1506  /* We are probably not using knet */
1507  if (!global_instance) {
1508  return CS_ERR_NOT_EXIST;
1509  }
1510 
1511  if (link_no >= INTERFACE_MAX) {
1512  return CS_ERR_NOT_EXIST; /* Invalid link number */
1513  }
1514 
1515  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1516  if (res) {
1517  switch (errno) {
1518  case EINVAL:
1519  ret = CS_ERR_INVALID_PARAM;
1520  break;
1521  case EBUSY:
1522  ret = CS_ERR_BUSY;
1523  break;
1524  case EDEADLK:
1525  ret = CS_ERR_TRY_AGAIN;
1526  break;
1527  default:
1528  ret = CS_ERR_LIBRARY;
1529  break;
1530  }
1531  }
1532 
1533  return (ret);
1534 }
1535 
1537  struct knet_handle_stats *stats)
1538 {
1539  int res;
1540 
1541  /* We are probably not using knet */
1542  if (!global_instance) {
1543  return CS_ERR_NOT_EXIST;
1544  }
1545 
1546  res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1547  if (res != 0) {
1548  return (qb_to_cs_error(-errno));
1549  }
1550 
1551  return CS_OK;
1552 }
1553 
1554 static void timer_function_merge_detect_timeout (
1555  void *data)
1556 {
1557  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1558 
1559  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1560  instance->send_merge_detect_message = 1;
1561  }
1562 
1564 
1565  totemknet_start_merge_detect_timeout(instance);
1566 }
1567 
1568 static void totemknet_start_merge_detect_timeout(
1569  void *knet_context)
1570 {
1571  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1572 
1573  qb_loop_timer_add(instance->poll_handle,
1574  QB_LOOP_MED,
1575  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1576  (void *)instance,
1577  timer_function_merge_detect_timeout,
1578  &instance->timer_merge_detect_timeout);
1579 
1580 }
1581 
1582 static void totemknet_stop_merge_detect_timeout(
1583  void *knet_context)
1584 {
1585  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1586 
1587  qb_loop_timer_del(instance->poll_handle,
1588  instance->timer_merge_detect_timeout);
1589 }
1590 
1591 static void log_flush_messages (void *knet_context)
1592 {
1593  struct pollfd pfd;
1594  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1595  int cont;
1596 
1597  cont = 1;
1598 
1599  while (cont) {
1600  pfd.fd = instance->logpipes[0];
1601  pfd.events = POLLIN;
1602  pfd.revents = 0;
1603 
1604  if ((poll(&pfd, 1, 0) > 0) &&
1605  (pfd.revents & POLLIN) &&
1606  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1607  cont = 1;
1608  } else {
1609  cont = 0;
1610  }
1611  }
1612 }
1613 
1614 
1615 #ifdef HAVE_LIBNOZZLE
1616 #define NOZZLE_NAME "nozzle.name"
1617 #define NOZZLE_IPADDR "nozzle.ipaddr"
1618 #define NOZZLE_PREFIX "nozzle.ipprefix"
1619 #define NOZZLE_MACADDR "nozzle.macaddr"
1620 
1621 #define NOZZLE_CHANNEL 1
1622 
1623 
1624 static char *get_nozzle_script_dir(void *knet_context)
1625 {
1626  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1627  char filename[PATH_MAX + FILENAME_MAX + 1];
1628  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1629  int res;
1630  const char *dirname_res;
1631 
1632  /*
1633  * Build script directory based on corosync.conf file location
1634  */
1635  res = snprintf(filename, sizeof(filename), "%s",
1637  if (res >= sizeof(filename)) {
1638  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1639  return NULL;
1640  }
1641 
1642  dirname_res = dirname(filename);
1643 
1644  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1645  dirname_res, "updown.d");
1646  if (res >= sizeof(updown_dirname)) {
1647  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1648  return NULL;
1649  }
1650  return updown_dirname;
1651 }
1652 
1653 /*
1654  * Deliberately doesn't return the status as caller doesn't care.
1655  * The result will be logged though
1656  */
1657 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1658 {
1659  int res;
1660  char *exec_string;
1661 
1662  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1663  if (res == -1 && errno != ENOENT) {
1664  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1665  } else if (res == -2) {
1666  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1667  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1668  }
1669 }
1670 
1671 /*
1672  * Reparse IP address to add in our node ID
1673  * IPv6 addresses must end in '::'
1674  * IPv4 addresses must just be valid
1675  * '/xx' lengths are optional for IPv6, mandatory for IPv4
1676  *
1677  * Returns the modified IP address as a string to pass into libnozzle
1678  */
1679 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1680  const char *input_addr,
1681  const char *prefix, int nodeid,
1682  char *output_addr, size_t output_len)
1683 {
1684  char *coloncolon;
1685  int bits;
1686  int max_prefix = 64;
1687  uint32_t nodeid_mask;
1688  uint32_t addr_mask;
1689  uint32_t masked_nodeid;
1690  struct in_addr *addr;
1691  struct totem_ip_address totemip;
1692 
1693  coloncolon = strstr(input_addr, "::");
1694  if (!coloncolon) {
1695  max_prefix = 30;
1696  }
1697 
1698  bits = atoi(prefix);
1699  if (bits < 8 || bits > max_prefix) {
1700  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1701  return -1;
1702  }
1703 
1704  /* IPv6 is easy */
1705  if (coloncolon) {
1706  memcpy(output_addr, input_addr, coloncolon-input_addr);
1707  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1708  return 0;
1709  }
1710 
1711  /* For IPv4 we need to parse the address into binary, mask off the required bits,
1712  * add in the masked_nodeid and 'print' it out again
1713  */
1714  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1715  addr_mask = UINT32_MAX ^ nodeid_mask;
1716  masked_nodeid = nodeid & nodeid_mask;
1717 
1718  if (totemip_parse(&totemip, input_addr, AF_INET)) {
1719  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1720  return -1;
1721  }
1722  addr = (struct in_addr *)&totemip.addr;
1723  addr->s_addr &= htonl(addr_mask);
1724  addr->s_addr |= htonl(masked_nodeid);
1725 
1726  inet_ntop(AF_INET, addr, output_addr, output_len);
1727  return 0;
1728 }
1729 
1730 static int create_nozzle_device(void *knet_context, const char *name,
1731  const char *ipaddr, const char *prefix,
1732  const char *macaddr)
1733 {
1734  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1735  char device_name[IFNAMSIZ+1];
1736  size_t size = IFNAMSIZ;
1737  int8_t channel = NOZZLE_CHANNEL;
1738  nozzle_t nozzle_dev;
1739  int nozzle_fd;
1740  int res;
1741  char *updown_dir;
1742  char parsed_ipaddr[INET6_ADDRSTRLEN];
1743  char mac[19];
1744 
1745  memset(device_name, 0, size);
1746  memset(&mac, 0, sizeof(mac));
1747  strncpy(device_name, name, size);
1748 
1749  updown_dir = get_nozzle_script_dir(knet_context);
1750  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
1751 
1752  nozzle_dev = nozzle_open(device_name, size, updown_dir);
1753  if (!nozzle_dev) {
1754  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
1755  return -1;
1756  }
1757  instance->nozzle_handle = nozzle_dev;
1758 
1759  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
1760  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
1761  goto out_clean;
1762  }
1763 
1764  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
1765  /* Prints its own errors */
1766  goto out_clean;
1767  }
1768  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
1769  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
1770  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
1771  goto out_clean;
1772  }
1773 
1774  nozzle_fd = nozzle_get_fd(nozzle_dev);
1775  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
1776 
1777  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
1778  if (res != 0) {
1779  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
1780  goto out_clean;
1781  }
1782 
1783  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
1784 
1785  res = nozzle_set_up(nozzle_dev);
1786  if (res != 0) {
1787  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
1788  goto out_clean;
1789  }
1790  run_nozzle_script(instance, NOZZLE_UP, "up");
1791 
1792  return 0;
1793 
1794 out_clean:
1795  nozzle_close(nozzle_dev);
1796  return -1;
1797 }
1798 
1799 static int remove_nozzle_device(void *knet_context)
1800 {
1801  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1802  int res;
1803  int datafd;
1804 
1805  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
1806  if (res != 0) {
1807  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1808  return -1;
1809  }
1810 
1811  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
1812  if (res != 0) {
1813  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1814  return -1;
1815  }
1816 
1817  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
1818  res = nozzle_set_down(instance->nozzle_handle);
1819  if (res != 0) {
1820  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
1821  return -1;
1822  }
1823  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
1824 
1825  res = nozzle_close(instance->nozzle_handle);
1826  if (res != 0) {
1827  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
1828  return -1;
1829  }
1830  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
1831  return 0;
1832 }
1833 
1834 static void free_nozzle(struct totemknet_instance *instance)
1835 {
1836  free(instance->nozzle_name);
1837  free(instance->nozzle_ipaddr);
1838  free(instance->nozzle_prefix);
1839  free(instance->nozzle_macaddr);
1840 
1841  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
1842  instance->nozzle_macaddr = NULL;
1843 }
1844 
1845 static int setup_nozzle(void *knet_context)
1846 {
1847  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1848  char *ipaddr_str = NULL;
1849  char *name_str = NULL;
1850  char *prefix_str = NULL;
1851  char *macaddr_str = NULL;
1852  char mac[32];
1853  int name_res;
1854  int macaddr_res;
1855  int res = -1;
1856 
1857  /*
1858  * Return value ignored on purpose. icmap_get_string changes
1859  * ipaddr_str/prefix_str only on success.
1860  */
1861  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
1862  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
1863  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
1864  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
1865 
1866  /* Is is being removed? */
1867  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
1868  remove_nozzle_device(instance);
1869  free_nozzle(instance);
1870  goto out_free;
1871  }
1872 
1873  if (!name_str) {
1874  /* no nozzle */
1875  goto out_free;
1876  }
1877 
1878  if (!ipaddr_str) {
1879  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
1880  goto out_free;
1881  }
1882 
1883  if (!prefix_str) {
1884  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
1885  goto out_free;
1886  }
1887 
1888  if (macaddr_str && strlen(macaddr_str) != 17) {
1889  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
1890  goto out_free;
1891  }
1892  if (!macaddr_str) {
1893  macaddr_str = (char*)"54:54:01:00:00:00";
1894  }
1895 
1896  if (instance->nozzle_name &&
1897  (strcmp(name_str, instance->nozzle_name) == 0) &&
1898  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
1899  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
1900  (instance->nozzle_macaddr == NULL ||
1901  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
1902  /* Nothing has changed */
1903  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
1904  goto out_free;
1905  }
1906 
1907  /* Add nodeid into MAC address */
1908  memcpy(mac, macaddr_str, 12);
1909  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
1910  instance->our_nodeid >> 8,
1911  instance->our_nodeid & 0xFF);
1912  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
1913 
1914  if (name_res == CS_OK && name_str) {
1915  /* Reconfigure */
1916  if (instance->nozzle_name) {
1917  remove_nozzle_device(instance);
1918  free_nozzle(instance);
1919  }
1920 
1921  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
1922  mac);
1923 
1924  instance->nozzle_name = strdup(name_str);
1925  instance->nozzle_ipaddr = strdup(ipaddr_str);
1926  instance->nozzle_prefix = strdup(prefix_str);
1927  instance->nozzle_macaddr = strdup(macaddr_str);
1928  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
1929  !instance->nozzle_prefix) {
1930  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
1931  /*
1932  * This 'free' will cause a complete reconfigure of the device next time we reload
1933  * but will also let the the current device keep working until then.
1934  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
1935  */
1936  free_nozzle(instance);
1937  }
1938  }
1939 
1940 out_free:
1941  free(name_str);
1942  free(ipaddr_str);
1943  free(prefix_str);
1944  if (macaddr_res == CS_OK) {
1945  free(macaddr_str);
1946  }
1947 
1948  return res;
1949 }
1950 #endif // HAVE_LIBNOZZLE
totemknet_ifaces_get
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:482
knet_log_printf
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:228
totem_logging_configuration::log_level_error
int log_level_error
Definition: totem.h:109
totemknet_instance::nozzle_prefix
char * nozzle_prefix
Definition: totemknet.c:180
totem_config::node_id
unsigned int node_id
Definition: totem.h:160
NOZZLE_CHANNEL
#define NOZZLE_CHANNEL
Definition: totemknet.c:1621
value
uint32_t value
Definition: exec/votequorum.c:101
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
totemknet_instance::nozzle_name
char * nozzle_name
Definition: totemknet.c:178
totemknet_instance::totemknet_log_level_error
int totemknet_log_level_error
Definition: totemknet.c:126
totemknet_mcast_flush_send
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1158
totemknet_instance::totemknet_iface_change_fn
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
totem_interface::knet_ping_precision
int knet_ping_precision
Definition: totem.h:92
NOZZLE_PREFIX
#define NOZZLE_PREFIX
Definition: totemknet.c:1618
totemknet_instance::totemknet_mtu_changed
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
MSG_NOSIGNAL
#define MSG_NOSIGNAL
Definition: totemknet.c:83
totemknet_member_add
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1285
work_item::msg_len
unsigned int msg_len
Definition: totemknet.c:191
stats_knet_del_member
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:628
totemknet_instance::nozzle_handle
nozzle_t nozzle_handle
Definition: totemknet.c:182
totemknet_buffer_release
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1124
totemknet_token_send
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1146
NOZZLE_MACADDR
#define NOZZLE_MACADDR
Definition: totemknet.c:1619
CS_ERR_LIBRARY
@ CS_ERR_LIBRARY
Definition: corotypes.h:99
CS_ERR_TRY_AGAIN
@ CS_ERR_TRY_AGAIN
Definition: corotypes.h:103
LOGSYS_LEVEL_ERROR
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
totemknet_crypto_set
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:358
totemknet_reconfigure
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1459
totemknet_iface_set
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1268
totem_config::knet_pmtud_interval
unsigned int knet_pmtud_interval
Definition: totem.h:162
work_item
Definition: totemknet.c:189
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
totem_logging_configuration::log_level_debug
int log_level_debug
Definition: totem.h:112
totemip_print
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:264
totemknet_net_mtu_adjust
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1195
totemsrp_stats_t
Definition: totemstats.h:53
type
char type
Definition: totem.h:4
totemknet_instance::poll_handle
qb_loop_t * poll_handle
Definition: totemknet.c:96
totem_logging_configuration::log_level_notice
int log_level_notice
Definition: totem.h:111
CS_ERR_NOT_EXIST
@ CS_ERR_NOT_EXIST
Definition: corotypes.h:109
totem_interface::knet_link_priority
int knet_link_priority
Definition: totem.h:89
KNET_LOGSYS_PERROR
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:244
totemknet_instance::totemknet_log_level_notice
int totemknet_log_level_notice
Definition: totemknet.c:130
totemknet_finalize
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:560
libknet_log_printf
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:236
stats_knet_add_member
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:618
totemknet_instance::totem_config
struct totem_config * totem_config
Definition: totemknet.c:161
totemknet_instance::context
void * context
Definition: totemknet.c:102
CFG_INTERFACE_STATUS_MAX_LEN
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
ICMAP_TRACK_PREFIX
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition: icmap.h:85
NOZZLE_NAME
#define NOZZLE_NAME
Definition: totemknet.c:1616
totem_ip_address::nodeid
unsigned int nodeid
Definition: coroapi.h:112
totemknet_processor_count_set
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1129
ENTER
#define ENTER
Definition: logsys.h:324
totemknet_instance::iov_buffer
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
corosync_get_config_file
const char * corosync_get_config_file(void)
Definition: main.c:204
totemknet_link_get_status
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1499
totem_config::merge_timeout
unsigned int merge_timeout
Definition: totem.h:190
CS_PRI_NODE_ID
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
totemknet_stats_clear
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1490
swab.h
totem_config::interfaces
struct totem_interface * interfaces
Definition: totem.h:158
totem_interface::knet_transport
int knet_transport
Definition: totem.h:94
totemknet_instance::totemknet_log_level_debug
int totemknet_log_level_debug
Definition: totemknet.c:132
addr
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
totem_config::knet_compression_level
int knet_compression_level
Definition: totem.h:228
totemip_copy
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
ICMAP_TRACK_ADD
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
icmap.h
header
struct totem_message_header header
Definition: totemsrp.c:2
totemknet_instance::nozzle_ipaddr
char * nozzle_ipaddr
Definition: totemknet.c:179
totemknet_instance::nozzle_macaddr
char * nozzle_macaddr
Definition: totemknet.c:181
totemknet_instance::totemknet_log_level_security
int totemknet_log_level_security
Definition: totemknet.c:124
CS_OK
@ CS_OK
Definition: corotypes.h:98
LOGSYS_LEVEL_INFO
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
totem_config::net_mtu
unsigned int net_mtu
Definition: totem.h:202
totemip.h
INTERFACE_MAX
#define INTERFACE_MAX
Definition: coroapi.h:88
totemknet_instance::my_ids
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
totem_config::knet_compression_threshold
uint32_t knet_compression_threshold
Definition: totem.h:226
CS_ERR_BUSY
@ CS_ERR_BUSY
Definition: corotypes.h:107
totem_interface::configured
uint8_t configured
Definition: totem.h:87
totemknet_instance::send_merge_detect_message
int send_merge_detect_message
Definition: totemknet.c:169
cs_error_t
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:97
__attribute__
typedef __attribute__
NOZZLE_IPADDR
#define NOZZLE_IPADDR
Definition: totemknet.c:1617
totem_interface::boundto
struct totem_ip_address boundto
Definition: totem.h:83
LEAVE
#define LEAVE
Definition: logsys.h:325
totemknet_instance::crypto_inst
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
icmap_track_add
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1159
totemknet_instance::knet_fd
int knet_fd
Definition: totemknet.c:174
totemknet_iface_check
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1185
totemknet_instance::timer_netif_check_timeout
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
totem_config::crypto_model
char * crypto_model
Definition: totem.h:218
totem_logging_configuration::log_subsys_id
int log_subsys_id
Definition: totem.h:114
totemknet_instance
Definition: totemknet.c:93
totem_config::link_mode
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:198
totemknet_instance::knet_context
void(*) void knet_context)
Definition: totemknet.c:145
totemknet_recv_flush
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1136
totemknet_instance::ip_port
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
totemknet_instance::our_nodeid
int our_nodeid
Definition: totemknet.c:157
totem_config
Definition: totem.h:152
qb_to_cs_error
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
totemknet_token_target_set
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1202
totemknet.h
totem_ip_address
The totem_ip_address struct.
Definition: coroapi.h:111
totemknet_instance::token_target
struct totem_ip_address token_target
Definition: totemknet.c:163
sq.h
icmap_get_uint32
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:892
totemknet_instance::log_mutex
pthread_mutex_t log_mutex
Definition: totemknet.c:176
totemknet_initialize
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:885
global_instance
struct totemknet_instance * global_instance
Definition: totemknet.c:187
totem_message_header
Definition: totem.h:124
totem_interface::knet_pong_count
int knet_pong_count
Definition: totem.h:93
totemknet_member_list_rebind_ip
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1453
totem_interface::ip_port
uint16_t ip_port
Definition: totem.h:85
totem_interface::knet_ping_interval
int knet_ping_interval
Definition: totem.h:90
_logsys_subsys_create
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
totemknet_instance::loopback_link
int loopback_link
Definition: totemknet.c:159
totemknet_instance::knet_handle
knet_handle_t knet_handle
Definition: totemknet.c:98
totemip_parse
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:314
ICMAP_TRACK_DELETE
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
work_item::msg
const void * msg
Definition: totemknet.c:190
totemknet_instance::timer_merge_detect_timeout
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
user_data
void * user_data
Definition: sam.c:127
system_from
struct srp_addr system_from
Definition: totemsrp.c:3
ICMAP_TRACK_MODIFY
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
totem_logging_configuration::log_level_warning
int log_level_warning
Definition: totem.h:110
totemknet_recv_mcast_empty
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1216
totemknet_instance::merge_detect_messages_sent_before_timeout
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
icmap_get_string
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:856
totemknet_member_remove
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1407
main.h
util.h
totemknet_send_flush
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1141
totemknet_mcast_noflush_send
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1171
totemknet_instance::knet_subsys_id
int knet_subsys_id
Definition: totemknet.c:136
totemknet_instance::totemknet_deliver_fn
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
totemknet_buffer_alloc
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1118
nodeid
unsigned int nodeid
Definition: coroapi.h:75
totem_interface::knet_ping_timeout
int knet_ping_timeout
Definition: totem.h:91
totem_config::private_key
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:167
PROCESSOR_COUNT_MAX
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
config.h
totemknet_handle_get_stats
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1536
CS_ERR_INVALID_PARAM
@ CS_ERR_INVALID_PARAM
Definition: corotypes.h:104
totemknet_instance::totemknet_subsys_id
int totemknet_subsys_id
Definition: totemknet.c:134
logsys.h
totem_config::crypto_hash_type
char * crypto_hash_type
Definition: totem.h:222
totemknet_instance::totemknet_target_set_completed
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
totem_logging_configuration::log_printf
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:99
LOGSYS_LEVEL_WARNING
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
totem_interface::bindnet
struct totem_ip_address bindnet
Definition: totem.h:82
icmap_track
Definition: icmap.c:61
totem_config::private_key_len
unsigned int private_key_len
Definition: totem.h:169
totemknet_instance::link_mode
int link_mode
Definition: totemknet.c:100
totem_config::crypto_cipher_type
char * crypto_cipher_type
Definition: totem.h:220
totemknet_instance::totemknet_log_printf
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
icmap_notify_value
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
stats_knet_add_handle
void stats_knet_add_handle(void)
Definition: stats.c:641
totemip_totemip_to_sockaddr_convert
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:272
totem_config::block_unlisted_ips
unsigned int block_unlisted_ips
Definition: totem.h:236
icmap_get_uint8
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:868
totemknet_instance::totemknet_log_level_warning
int totemknet_log_level_warning
Definition: totemknet.c:128
totemknet_instance::logpipes
int logpipes[2]
Definition: totemknet.c:173
totemstats.h
totem_message_header::target_nodeid
unsigned int target_nodeid
Definition: totem.h:130
work_item::instance
struct totemknet_instance * instance
Definition: totemknet.c:192
totemknet_instance::link_status
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
totem_config::knet_compression_model
char * knet_compression_model
Definition: totem.h:224
totem_config::totem_logging_configuration
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:200