76 #include <sys/types.h>
81 #include <sys/socket.h>
84 #include <sys/resource.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
96 #include <semaphore.h>
99 #ifdef HAVE_LIBSYSTEMD
100 #include <systemd/sd-daemon.h>
103 #include <qb/qbdefs.h>
104 #include <qb/qblog.h>
105 #include <qb/qbloop.h>
106 #include <qb/qbutil.h>
107 #include <qb/qbipcs.h>
130 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
131 #define IPC_LOGSYS_SIZE 1024*64
133 #define IPC_LOGSYS_SIZE 8192*128
148 #define SERVER_BACKLOG 5
150 static int sched_priority = 0;
152 static unsigned int service_count = 32;
158 static int sync_in_process = 1;
160 static qb_loop_t *corosync_poll_handle;
166 static const char *corosync_lock_file =
LOCALSTATEDIR"/run/corosync.pid";
168 static char corosync_config_file[PATH_MAX + 1] =
COROSYSCONFDIR "/corosync.conf";
172 return (corosync_poll_handle);
180 int (*dispatch_fn) (
int fd,
184 return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
190 return qb_loop_poll_del(handle, fd);
207 return (corosync_config_file);
210 static void corosync_blackbox_write_to_file (
void)
212 char fname[PATH_MAX];
213 char fdata_fname[PATH_MAX];
214 char time_str[PATH_MAX];
215 struct tm cur_time_tm;
219 cur_time_t = time(NULL);
220 localtime_r(&cur_time_t, &cur_time_tm);
222 strftime(time_str, PATH_MAX,
"%Y-%m-%dT%H:%M:%S", &cur_time_tm);
223 if (snprintf(fname, PATH_MAX,
"%s/fdata-%s-%lld",
226 (
long long int)getpid()) >= PATH_MAX) {
231 if ((res = qb_log_blackbox_write_to_file(fname)) < 0) {
235 snprintf(fdata_fname,
sizeof(fdata_fname),
"%s/fdata",
get_state_dir());
237 if (symlink(fname, fdata_fname) == -1) {
243 static void unlink_all_completed (
void)
246 qb_loop_stop (corosync_poll_handle);
255 static int32_t sig_diag_handler (
int num,
void *data)
261 static int32_t sig_exit_handler (
int num,
void *data)
268 static void sigsegv_handler (
int num)
270 (void)signal (num, SIG_DFL);
271 corosync_blackbox_write_to_file ();
276 #define LOCALHOST_IP inet_addr("127.0.0.1")
278 static void *corosync_group_handle;
285 static void serialize_lock (
void)
289 static void serialize_unlock (
void)
293 static void corosync_sync_completed (
void)
296 "Completed service synchronization, ready to provide service.");
306 #ifdef HAVE_LIBSYSTEMD
307 sd_notify (0,
"READY=1");
311 static int corosync_sync_callbacks_retrieve (
319 if (callbacks == NULL) {
334 static void member_object_joined (
unsigned int nodeid)
341 "runtime.members.%u.ip",
nodeid);
343 "runtime.members.%u.join_count",
nodeid);
345 "runtime.members.%u.status",
nodeid);
360 static void member_object_left (
unsigned int nodeid)
365 "runtime.members.%u.status",
nodeid);
372 static void confchg_fn (
374 const unsigned int *member_list,
size_t member_list_entries,
375 const unsigned int *left_list,
size_t left_list_entries,
376 const unsigned int *joined_list,
size_t joined_list_entries,
380 int abort_activate = 0;
382 if (sync_in_process == 1) {
389 for (i = 0; i < left_list_entries; i++) {
390 member_object_left (left_list[i]);
392 for (i = 0; i < joined_list_entries; i++) {
393 member_object_joined (joined_list[i]);
398 for (i = 0; i < service_count; i++) {
401 member_list, member_list_entries,
402 left_list, left_list_entries,
403 joined_list, joined_list_entries,
ring_id);
407 if (abort_activate) {
418 static void priv_drop (
void)
423 static void corosync_tty_detach (
void)
451 devnull = open(
"/dev/null", O_RDWR);
456 if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
457 || dup2(devnull, 2) < 0) {
464 static void corosync_mlockall (
void)
467 struct rlimit rlimit;
469 rlimit.rlim_cur = RLIM_INFINITY;
470 rlimit.rlim_max = RLIM_INFINITY;
472 #ifndef RLIMIT_MEMLOCK
473 #define RLIMIT_MEMLOCK RLIMIT_VMEM
479 "Could not increase RLIMIT_MEMLOCK, not locking memory");
483 res = mlockall (MCL_CURRENT | MCL_FUTURE);
486 "Could not lock memory of service to avoid page faults");
491 static void corosync_totem_stats_updater (
void *data)
494 uint32_t total_mtt_rx_token;
495 uint32_t total_backlog_calc;
496 uint32_t total_token_holdtime;
511 cstr =
"number of multicast sendmsg failures is above threshold";
515 cstr =
"totem is continuously in gather state";
519 "Totem is unable to form a cluster because of an "
520 "operating system or network fault (reason: %s). The most common "
521 "cause of this message is that the local firewall is "
522 "configured improperly.", cstr);
528 total_mtt_rx_token = 0;
529 total_token_holdtime = 0;
530 total_backlog_calc = 0;
562 corosync_totem_stats_updater,
563 &corosync_stats_timer_handle);
566 static void corosync_totem_stats_init (
void)
570 corosync_totem_stats_updater,
571 &corosync_stats_timer_handle);
574 static void deliver_fn (
577 unsigned int msg_len,
578 int endian_conversion_required)
580 const struct qb_ipc_request_header *
header;
586 if (endian_conversion_required) {
609 if (endian_conversion_required) {
610 assert(
corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
620 const struct iovec *iovec,
621 unsigned int iov_len,
624 const struct qb_ipc_request_header *req = iovec->iov_base;
628 service = req->id >> 16;
629 fn_id = req->id & 0xffff;
638 static void corosync_ring_id_create_or_load (
644 char filename[PATH_MAX];
646 snprintf (filename,
sizeof(filename),
"%s/ringid_%u",
648 fd = open (filename, O_RDONLY, 0700);
659 if ((fd == -1) || (res !=
sizeof (uint64_t))) {
662 fd = open (filename, O_CREAT|O_RDWR, 0700);
668 "Couldn't write ringid file '%s'", filename);
674 "Couldn't create ringid file '%s'", filename);
683 static void corosync_ring_id_store (
687 char filename[PATH_MAX];
691 snprintf (filename,
sizeof(filename),
"%s/ringid_%u",
694 fd = open (filename, O_WRONLY, 0700);
696 fd = open (filename, O_CREAT|O_RDWR, 0700);
718 static qb_loop_timer_handle recheck_the_q_level_timer;
734 unsigned int service,
737 void *sending_allowed_private_data)
741 struct iovec reserve_iovec;
742 struct qb_ipc_request_header *
header = (
struct qb_ipc_request_header *)msg;
745 reserve_iovec.iov_base = (
char *)
header;
746 reserve_iovec.iov_len =
header->size;
749 corosync_group_handle,
760 sending_allowed = QB_FALSE;
766 sending_allowed = QB_TRUE;
768 sending_allowed = QB_TRUE;
775 return -EHOSTUNREACH;
778 return (sending_allowed);
796 assert (source != NULL);
807 assert ((source != NULL) && (conn != NULL));
820 static void timer_function_scheduler_timeout (
void *data)
823 unsigned long long tv_current;
824 unsigned long long tv_diff;
826 tv_current = qb_util_nano_current_get ();
828 if (timeout_data->
tv_prev == 0) {
832 timeout_data->
tv_prev = tv_current;
836 tv_diff = tv_current - timeout_data->
tv_prev;
837 timeout_data->
tv_prev = tv_current;
841 "(threshold is %0.4f ms). Consider token timeout increase.",
842 (
float)tv_diff / QB_TIME_NS_IN_MSEC, (
float)timeout_data->
max_tv_diff / QB_TIME_NS_IN_MSEC);
849 qb_loop_timer_add (corosync_poll_handle,
853 timer_function_scheduler_timeout,
858 static int corosync_set_rr_scheduler (
void)
862 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
865 sched_priority = sched_get_priority_max (SCHED_RR);
866 if (sched_priority != -1) {
871 "Could not set SCHED_RR at priority %d",
875 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
876 qb_log_thread_priority_set (SCHED_OTHER, 0);
884 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
885 res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
891 "Could not set logsys thread priority."
892 " Can't continue because of priority inversions.");
898 "Could not get maximum scheduler priority");
904 "The Platform is missing process priority setting features. Leaving at default.");
914 static const char *corosync_basename(
const char *file_name)
917 base = strrchr (file_name,
'/');
926 _logsys_log_printf(
int level,
int subsys,
927 const char *function_name,
928 const char *file_name,
934 _logsys_log_printf(
int level,
int subsys,
935 const
char *function_name,
936 const
char *file_name,
938 const
char *format, ...)
942 va_start(ap, format);
943 qb_log_from_external_source_va(function_name, corosync_basename(file_name),
944 format, level, file_line,
949 static void fplay_key_change_notify_fn (
951 const char *key_name,
956 if (strcmp(key_name,
"runtime.blackbox.dump_flight_data") == 0) {
957 fprintf(stderr,
"Writetofile\n");
958 corosync_blackbox_write_to_file ();
960 if (strcmp(key_name,
"runtime.blackbox.dump_state") == 0) {
961 fprintf(stderr,
"statefump\n");
966 static void corosync_fplay_control_init (
void)
975 fplay_key_change_notify_fn,
979 fplay_key_change_notify_fn,
983 static void force_gather_notify_fn(
985 const char *key_name,
997 if (strcmp(key_name,
"runtime.force_gather") == 0) {
1006 static void corosync_force_gather_init (
void)
1014 force_gather_notify_fn,
1025 static void set_icmap_ro_keys_flag (
void)
1060 static void main_service_ready (
void)
1073 corosync_totem_stats_init ();
1074 corosync_fplay_control_init ();
1075 corosync_force_gather_init ();
1078 corosync_sync_callbacks_retrieve,
1079 corosync_sync_completed);
1082 static enum e_corosync_done corosync_flock (
const char *lockfile, pid_t pid)
1092 lf = open (lockfile, O_WRONLY | O_CREAT, 0640);
1099 lock.l_type = F_WRLCK;
1101 lock.l_whence = SEEK_SET;
1103 if (fcntl (lf, F_SETLK, &
lock) == -1) {
1123 if (ftruncate (lf, 0) == -1) {
1127 goto error_close_unlink;
1130 memset (pid_s, 0,
sizeof (pid_s));
1131 snprintf (pid_s,
sizeof (pid_s) - 1,
"%u\n", pid);
1134 if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1135 if (errno == EINTR) {
1139 "Error was %s", strerror (errno));
1141 goto error_close_unlink;
1145 if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) {
1147 "Error was %s", strerror (errno));
1149 goto error_close_unlink;
1151 fd_flag |= FD_CLOEXEC;
1152 if (fcntl (lf, F_SETFD, fd_flag) == -1) {
1154 "Error was %s", strerror (errno));
1156 goto error_close_unlink;
1169 static int corosync_move_to_root_cgroup(
void) {
1180 f = fopen(
"/sys/fs/cgroup/cpu/cpu.rt_runtime_us",
"rt");
1183 "system without cgroup or with disabled CONFIG_RT_GROUP_SCHED");
1190 f = fopen(
"/sys/fs/cgroup/cpu/tasks",
"w");
1197 if (fprintf(f,
"%jd\n", (intmax_t)getpid()) <= 0) {
1200 goto close_and_exit_res;
1204 if (fclose(f) != 0) {
1215 int main (
int argc,
char **argv,
char **envp)
1217 const char *error_string;
1220 int background, sched_rr, prio, testonly, move_to_root_cgroup;
1221 struct stat stat_out;
1223 uint64_t totem_config_warnings;
1228 int log_subsys_id_totem;
1235 while ((ch = getopt (argc, argv,
"c:ftv")) != EOF) {
1239 res = snprintf(corosync_config_file,
sizeof(corosync_config_file),
"%s", optarg);
1240 if (res >=
sizeof(corosync_config_file)) {
1241 fprintf (stderr,
"Config file path too long.\n");
1245 return EXIT_FAILURE;
1255 printf (
"Corosync Cluster Engine, version '%s'\n",
VERSION);
1256 printf (
"Copyright (c) 2006-2018 Red Hat, Inc.\n");
1258 return EXIT_SUCCESS;
1264 " -c : Corosync config file path.\n"\
1265 " -f : Start application in foreground.\n"\
1266 " -t : Test configuration and exit.\n"\
1267 " -v : Display version and SVN revision of Corosync and exit.\n");
1269 return EXIT_FAILURE;
1277 (void)signal (SIGSEGV, sigsegv_handler);
1278 (void)signal (SIGABRT, sigsegv_handler);
1279 #if MSG_NOSIGNAL != 0
1280 (void)signal (SIGPIPE, SIG_IGN);
1284 fprintf (stderr,
"Corosync Executive couldn't initialize configuration component.\n");
1285 syslog (
LOGSYS_LEVEL_ERROR,
"Corosync Executive couldn't initialize configuration component.");
1288 set_icmap_ro_keys_flag();
1301 fprintf (stderr,
"%s\n", error_string);
1307 fprintf (stderr,
"Corosync Executive couldn't initialize statistics component.\n");
1308 syslog (
LOGSYS_LEVEL_ERROR,
"Corosync Executive couldn't initialize statistics component.");
1322 fprintf(stderr,
"%s", error_string);
1336 "totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
1337 "totempg.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c");
1343 if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) {
1351 "Please make sure it has correct context and rights.",
get_state_dir());
1375 "Nodelist one is going to be used.");
1378 if (totem_config_warnings != 0) {
1399 move_to_root_cgroup = 1;
1401 if (strcmp(tmp_str,
"yes") != 0) {
1402 move_to_root_cgroup = 0;
1411 if (move_to_root_cgroup) {
1412 (void)corosync_move_to_root_cgroup();
1417 if (strcmp(tmp_str,
"yes") != 0) {
1425 if (strcmp(tmp_str,
"max") == 0) {
1427 }
else if (strcmp(tmp_str,
"min") == 0) {
1432 tmpli = strtol(tmp_str, &ep, 10);
1433 if (errno != 0 || *ep !=
'\0' || tmpli > INT_MAX || tmpli < INT_MIN) {
1448 if (corosync_set_rr_scheduler () != 0) {
1456 if (setpriority(PRIO_PGRP, 0, prio) != 0) {
1458 "Could not set priority %d", prio);
1484 corosync_tty_detach ();
1495 corosync_mlockall ();
1497 corosync_poll_handle = qb_loop_create ();
1503 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1504 SIGUSR2, NULL, sig_diag_handler, NULL);
1505 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1506 SIGINT, NULL, sig_exit_handler, NULL);
1507 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1508 SIGQUIT, NULL, sig_exit_handler, NULL);
1509 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1510 SIGTERM, NULL, sig_exit_handler, NULL);
1517 if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) !=
COROSYNC_DONE_EXIT) {
1532 corosync_poll_handle,
1540 main_service_ready);
1543 &corosync_group_handle,
1548 corosync_group_handle,
1569 qb_loop_run (corosync_poll_handle);
1579 qb_loop_destroy (corosync_poll_handle);
1588 unlink (corosync_lock_file);
1592 return EXIT_SUCCESS;