corosync  3.0.2
vsf_quorum.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2008-2015 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7  *
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of Red Hat Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <pwd.h>
38 #include <grp.h>
39 #include <sys/types.h>
40 #include <sys/poll.h>
41 #include <sys/uio.h>
42 #include <sys/mman.h>
43 #include <sys/socket.h>
44 #include <sys/un.h>
45 #include <sys/time.h>
46 #include <sys/resource.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 
57 #include "quorum.h"
58 #include <corosync/corotypes.h>
59 #include <qb/qbipc_common.h>
60 #include <corosync/corodefs.h>
61 #include <corosync/swab.h>
62 #include <qb/qblist.h>
63 #include <corosync/mar_gen.h>
64 #include <corosync/ipc_quorum.h>
65 #include <corosync/coroapi.h>
66 #include <corosync/logsys.h>
67 #include <corosync/icmap.h>
68 
69 #include "service.h"
70 #include "votequorum.h"
71 #include "vsf_ykd.h"
72 
73 LOGSYS_DECLARE_SUBSYS ("QUORUM");
74 
75 struct quorum_pd {
76  unsigned char track_flags;
77  int tracking_enabled;
78  struct qb_list_head list;
79  void *conn;
80 };
81 
83  struct qb_list_head list;
85  void *context;
86 };
87 
88 static void message_handler_req_lib_quorum_getquorate (void *conn,
89  const void *msg);
90 static void message_handler_req_lib_quorum_trackstart (void *conn,
91  const void *msg);
92 static void message_handler_req_lib_quorum_trackstop (void *conn,
93  const void *msg);
94 static void message_handler_req_lib_quorum_gettype (void *conn,
95  const void *msg);
96 static void send_library_notification(void *conn);
97 static void send_internal_notification(void);
98 static char *quorum_exec_init_fn (struct corosync_api_v1 *api);
99 static int quorum_lib_init_fn (void *conn);
100 static int quorum_lib_exit_fn (void *conn);
101 
102 static int primary_designated = 0;
103 static int quorum_type = 0;
104 static struct corosync_api_v1 *corosync_api;
105 static struct qb_list_head lib_trackers_list;
106 static struct qb_list_head internal_trackers_list;
107 static struct memb_ring_id quorum_ring_id;
108 static size_t quorum_view_list_entries = 0;
109 static int quorum_view_list[PROCESSOR_COUNT_MAX];
110 struct quorum_services_api_ver1 *quorum_iface = NULL;
111 static char view_buf[64];
112 
113 static void log_view_list(const unsigned int *view_list, size_t view_list_entries)
114 {
115  int total = (int)view_list_entries;
116  int len, pos, ret;
117  int i = 0;
118 
119  while (1) {
120  len = sizeof(view_buf);
121  pos = 0;
122  memset(view_buf, 0, len);
123 
124  for (; i < total; i++) {
125  ret = snprintf(view_buf + pos, len - pos, " %u", view_list[i]);
126  if (ret >= len - pos)
127  break;
128  pos += ret;
129  }
130  log_printf (LOGSYS_LEVEL_NOTICE, "Members[%d]:%s%s",
131  total, view_buf, i < total ? "\\" : "");
132 
133  if (i == total)
134  break;
135  }
136 }
137 
138 /* Internal quorum API function */
139 static void quorum_api_set_quorum(const unsigned int *view_list,
140  size_t view_list_entries,
141  int quorum, struct memb_ring_id *ring_id)
142 {
143  int old_quorum = primary_designated;
144  primary_designated = quorum;
145 
146  if (primary_designated && !old_quorum) {
147  log_printf (LOGSYS_LEVEL_NOTICE, "This node is within the primary component and will provide service.");
148  } else if (!primary_designated && old_quorum) {
149  log_printf (LOGSYS_LEVEL_NOTICE, "This node is within the non-primary component and will NOT provide any services.");
150  }
151 
152  quorum_view_list_entries = view_list_entries;
153  memcpy(&quorum_ring_id, ring_id, sizeof (quorum_ring_id));
154  memcpy(quorum_view_list, view_list, sizeof(unsigned int)*view_list_entries);
155 
156  log_view_list(view_list, view_list_entries);
157 
158  /* Tell internal listeners */
159  send_internal_notification();
160 
161  /* Tell IPC listeners */
162  send_library_notification(NULL);
163 }
164 
165 static struct corosync_lib_handler quorum_lib_service[] =
166 {
167  { /* 0 */
168  .lib_handler_fn = message_handler_req_lib_quorum_getquorate,
169  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
170  },
171  { /* 1 */
172  .lib_handler_fn = message_handler_req_lib_quorum_trackstart,
173  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
174  },
175  { /* 2 */
176  .lib_handler_fn = message_handler_req_lib_quorum_trackstop,
177  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
178  },
179  { /* 3 */
180  .lib_handler_fn = message_handler_req_lib_quorum_gettype,
181  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
182  }
183 };
184 
185 static struct corosync_service_engine quorum_service_handler = {
186  .name = "corosync cluster quorum service v0.1",
187  .id = QUORUM_SERVICE,
188  .priority = 1,
189  .private_data_size = sizeof (struct quorum_pd),
190  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED,
191  .allow_inquorate = CS_LIB_ALLOW_INQUORATE,
192  .lib_init_fn = quorum_lib_init_fn,
193  .lib_exit_fn = quorum_lib_exit_fn,
194  .lib_engine = quorum_lib_service,
195  .exec_init_fn = quorum_exec_init_fn,
196  .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler)
197 };
198 
200 {
201  return (&quorum_service_handler);
202 }
203 
204 /* -------------------------------------------------- */
205 
206 
207 /*
208  * Internal API functions for corosync
209  */
210 
211 static int quorum_quorate(void)
212 {
213  return primary_designated;
214 }
215 
216 
217 static int quorum_register_callback(quorum_callback_fn_t function, void *context)
218 {
219  struct internal_callback_pd *pd = malloc(sizeof(struct internal_callback_pd));
220  if (!pd)
221  return -1;
222 
223  pd->context = context;
224  pd->callback = function;
225  qb_list_add (&pd->list, &internal_trackers_list);
226 
227  return 0;
228 }
229 
230 static int quorum_unregister_callback(quorum_callback_fn_t function, void *context)
231 {
232  struct internal_callback_pd *pd;
233  struct qb_list_head *tmp, *tmp_iter;
234 
235  qb_list_for_each_safe(tmp, tmp_iter, &internal_trackers_list) {
236  pd = qb_list_entry(tmp, struct internal_callback_pd, list);
237  if (pd->callback == function && pd->context == context) {
238  qb_list_del(&pd->list);
239  free(pd);
240  return 0;
241  }
242  }
243  return -1;
244 }
245 
246 static struct quorum_callin_functions callins = {
247  .quorate = quorum_quorate,
248  .register_callback = quorum_register_callback,
249  .unregister_callback = quorum_unregister_callback
250 };
251 
252 /* --------------------------------------------------------------------- */
253 
254 static char *quorum_exec_init_fn (struct corosync_api_v1 *api)
255 {
256  char *quorum_module = NULL;
257  char *error;
258 
259  corosync_api = api;
260  qb_list_init (&lib_trackers_list);
261  qb_list_init (&internal_trackers_list);
262 
263  /*
264  * Tell corosync we have a quorum engine.
265  */
266  api->quorum_initialize(&callins);
267 
268  /*
269  * Look for a quorum provider
270  */
271  if (icmap_get_string("quorum.provider", &quorum_module) == CS_OK) {
273  "Using quorum provider %s", quorum_module);
274 
275  error = (char *)"Invalid quorum provider";
276 
277  if (strcmp (quorum_module, "corosync_votequorum") == 0) {
278  error = votequorum_init (api, quorum_api_set_quorum);
279  quorum_type = 1;
280  }
281  if (strcmp (quorum_module, "corosync_ykd") == 0) {
282  error = ykd_init (api, quorum_api_set_quorum);
283  quorum_type = 1;
284  }
285  if (error) {
287  "Quorum provider: %s failed to initialize.",
288  quorum_module);
289  free(quorum_module);
290  return (error);
291  }
292  }
293 
294  if (quorum_module) {
295  free(quorum_module);
296  quorum_module = NULL;
297  }
298 
299  /*
300  * setting quorum_type and primary_designated in the right order is important
301  * always try to lookup/init a quorum module, then revert back to be quorate
302  */
303 
304  if (quorum_type == 0) {
305  primary_designated = 1;
306  }
307 
308  return (NULL);
309 }
310 
311 static int quorum_lib_init_fn (void *conn)
312 {
313  struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
314 
315  log_printf(LOGSYS_LEVEL_DEBUG, "lib_init_fn: conn=%p", conn);
316 
317  qb_list_init (&pd->list);
318  pd->conn = conn;
319 
320  return (0);
321 }
322 
323 static int quorum_lib_exit_fn (void *conn)
324 {
325  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
326 
327  log_printf(LOGSYS_LEVEL_DEBUG, "lib_exit_fn: conn=%p", conn);
328 
330  qb_list_del (&quorum_pd->list);
331  qb_list_init (&quorum_pd->list);
332  }
333  return (0);
334 }
335 
336 
337 static void send_internal_notification(void)
338 {
339  struct qb_list_head *tmp;
340  struct internal_callback_pd *pd;
341 
342  qb_list_for_each(tmp, &internal_trackers_list) {
343  pd = qb_list_entry(tmp, struct internal_callback_pd, list);
344 
345  pd->callback(primary_designated, pd->context);
346  }
347 }
348 
349 static void send_library_notification(void *conn)
350 {
351  int size = sizeof(struct res_lib_quorum_notification) + sizeof(unsigned int)*quorum_view_list_entries;
352  char buf[size];
354  struct qb_list_head *tmp;
355  int i;
356 
357  log_printf(LOGSYS_LEVEL_DEBUG, "sending quorum notification to %p, length = %d", conn, size);
358 
359  res_lib_quorum_notification->quorate = primary_designated;
360  res_lib_quorum_notification->ring_seq = quorum_ring_id.seq;
361  res_lib_quorum_notification->view_list_entries = quorum_view_list_entries;
362  for (i=0; i<quorum_view_list_entries; i++) {
363  res_lib_quorum_notification->view_list[i] = quorum_view_list[i];
364  }
365 
367  res_lib_quorum_notification->header.size = size;
368  res_lib_quorum_notification->header.error = CS_OK;
369 
370  /* Send it to all interested parties */
371  if (conn) {
372  corosync_api->ipc_dispatch_send(conn, res_lib_quorum_notification, size);
373  }
374  else {
375  struct quorum_pd *qpd;
376 
377  qb_list_for_each(tmp, &lib_trackers_list) {
378  qpd = qb_list_entry(tmp, struct quorum_pd, list);
379 
380  corosync_api->ipc_dispatch_send(qpd->conn,
382  }
383  }
384  return;
385 }
386 
387 static void message_handler_req_lib_quorum_getquorate (void *conn,
388  const void *msg)
389 {
391 
392  log_printf(LOGSYS_LEVEL_DEBUG, "got quorate request on %p", conn);
393 
394  /* send status */
395  res_lib_quorum_getquorate.quorate = primary_designated;
398  res_lib_quorum_getquorate.header.error = CS_OK;
400 }
401 
402 static void message_handler_req_lib_quorum_trackstart (void *conn,
403  const void *msg)
404 {
406  struct qb_ipc_response_header res;
407  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
408  cs_error_t error = CS_OK;
409 
410  log_printf(LOGSYS_LEVEL_DEBUG, "got trackstart request on %p", conn);
411 
412  /*
413  * If an immediate listing of the current cluster membership
414  * is requested, generate membership list
415  */
418  log_printf(LOGSYS_LEVEL_DEBUG, "sending initial status to %p", conn);
419  send_library_notification(conn);
420  }
421 
423  error = CS_ERR_EXIST;
424  goto response_send;
425  }
426 
427  /*
428  * Record requests for tracking
429  */
432 
435 
436  qb_list_add (&quorum_pd->list, &lib_trackers_list);
437  }
438 
439 response_send:
440  /* send status */
441  res.size = sizeof(res);
443  res.error = error;
444  corosync_api->ipc_response_send(conn, &res, sizeof(struct qb_ipc_response_header));
445 }
446 
447 static void message_handler_req_lib_quorum_trackstop (void *conn, const void *msg)
448 {
449  struct qb_ipc_response_header res;
450  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
451 
452  log_printf(LOGSYS_LEVEL_DEBUG, "got trackstop request on %p", conn);
453 
455  res.error = CS_OK;
457  qb_list_del (&quorum_pd->list);
458  qb_list_init (&quorum_pd->list);
459  } else {
460  res.error = CS_ERR_NOT_EXIST;
461  }
462 
463  /* send status */
464  res.size = sizeof(res);
466  res.error = CS_OK;
467  corosync_api->ipc_response_send(conn, &res, sizeof(struct qb_ipc_response_header));
468 }
469 
470 static void message_handler_req_lib_quorum_gettype (void *conn,
471  const void *msg)
472 {
474 
475  log_printf(LOGSYS_LEVEL_DEBUG, "got quorum_type request on %p", conn);
476 
477  /* send status */
479  res_lib_quorum_gettype.header.size = sizeof(res_lib_quorum_gettype);
481  res_lib_quorum_gettype.header.error = CS_OK;
482  corosync_api->ipc_response_send(conn, &res_lib_quorum_gettype, sizeof(res_lib_quorum_gettype));
483 }
484 
corosync_api_v1
The corosync_api_v1 struct.
Definition: coroapi.h:225
MESSAGE_RES_QUORUM_TRACKSTOP
Definition: ipc_quorum.h:56
LOGSYS_DECLARE_SUBSYS
LOGSYS_DECLARE_SUBSYS("QUORUM")
CS_LIB_ALLOW_INQUORATE
Definition: coroapi.h:164
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
log_printf
#define log_printf(level, format, args...)
Definition: logsys.h:323
CS_TRACK_CURRENT
#define CS_TRACK_CURRENT
Definition: corotypes.h:87
ipc_quorum.h
coroapi.h
votequorum_init
char * votequorum_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t q_set_quorate_fn)
Definition: exec/votequorum.c:2455
MESSAGE_RES_QUORUM_GETQUORATE
Definition: ipc_quorum.h:54
service.h
quorum_callin_functions
The quorum_callin_functions struct.
Definition: coroapi.h:204
quorum_pd
Definition: exec/votequorum.c:222
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
internal_callback_pd::callback
quorum_callback_fn_t callback
Definition: vsf_quorum.c:84
CS_ERR_NOT_EXIST
Definition: corotypes.h:106
res_lib_quorum_gettype
The res_lib_quorum_gettype struct.
Definition: ipc_quorum.h:91
quorum_pd::track_flags
unsigned char track_flags
Definition: exec/votequorum.c:223
req_lib_quorum_trackstart
The req_lib_quorum_trackstart struct.
Definition: ipc_quorum.h:64
swab.h
corosync_api_v1::ipc_response_send
int(* ipc_response_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:258
quorum_pd::list
struct qb_list_head list
Definition: exec/votequorum.c:226
CS_LIB_FLOW_CONTROL_NOT_REQUIRED
Definition: coroapi.h:153
vsf_quorum_get_service_engine_ver0
struct corosync_service_engine * vsf_quorum_get_service_engine_ver0(void)
Definition: vsf_quorum.c:199
res_lib_quorum_gettype::quorum_type
mar_uint32_t quorum_type
Definition: ipc_quorum.h:93
corosync_lib_handler
The corosync_lib_handler struct.
Definition: coroapi.h:467
corosync_service_engine
The corosync_service_engine struct.
Definition: coroapi.h:490
icmap.h
internal_callback_pd
Definition: vsf_quorum.c:82
CS_OK
Definition: corotypes.h:95
res_lib_quorum_getquorate
The res_lib_quorum_getquorate struct.
Definition: ipc_quorum.h:72
mar_gen.h
quorum_pd::tracking_enabled
int tracking_enabled
Definition: exec/votequorum.c:224
internal_callback_pd::context
void * context
Definition: vsf_quorum.c:85
cs_error_t
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:94
quorum_callback_fn_t
void(* quorum_callback_fn_t)(int quorate, void *context)
The quorum_callback_fn_t callback.
Definition: coroapi.h:199
quorum_pd::conn
void * conn
Definition: exec/votequorum.c:227
MESSAGE_RES_QUORUM_NOTIFICATION
Definition: ipc_quorum.h:57
res_lib_quorum_getquorate::quorate
mar_uint32_t quorate
Definition: ipc_quorum.h:74
LOGSYS_LEVEL_NOTICE
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:74
CS_ERR_EXIST
Definition: corotypes.h:108
res_lib_quorum_notification::view_list
mar_uint32_t view_list[]
Definition: ipc_quorum.h:85
memb_ring_id
The memb_ring_id struct.
Definition: coroapi.h:122
ykd_init
char * ykd_init(struct corosync_api_v1 *corosync_api, quorum_set_quorate_fn_t set_primary)
Definition: vsf_ykd.c:511
req_lib_quorum_trackstart::track_flags
unsigned int track_flags
Definition: ipc_quorum.h:66
vsf_ykd.h
icmap_get_string
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:880
quorum.h
corosync_service_engine::name
const char * name
Definition: coroapi.h:491
CS_TRACK_CHANGES
#define CS_TRACK_CHANGES
Definition: corotypes.h:88
memb_ring_id::seq
unsigned long long seq
Definition: coroapi.h:124
corosync_lib_handler::lib_handler_fn
void(* lib_handler_fn)(void *conn, const void *msg)
Definition: coroapi.h:468
ring_id
struct memb_ring_id ring_id
Definition: totemsrp.c:264
quorum_callin_functions::quorate
int(* quorate)(void)
Definition: coroapi.h:205
PROCESSOR_COUNT_MAX
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
config.h
CS_TRACK_CHANGES_ONLY
#define CS_TRACK_CHANGES_ONLY
Definition: corotypes.h:89
MESSAGE_RES_QUORUM_GETTYPE
Definition: ipc_quorum.h:58
logsys.h
corotypes.h
QUORUM_SERVICE
Definition: corodefs.h:47
corosync_api_v1::quorum_initialize
int(* quorum_initialize)(struct quorum_callin_functions *fns)
Definition: coroapi.h:392
quorum_iface
struct quorum_services_api_ver1 * quorum_iface
Definition: vsf_quorum.c:110
corosync_api_v1::ipc_dispatch_send
int(* ipc_dispatch_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:263
votequorum.h
MESSAGE_RES_QUORUM_TRACKSTART
Definition: ipc_quorum.h:55
corosync_api_v1::ipc_private_data_get
void *(* ipc_private_data_get)(void *conn)
Definition: coroapi.h:256
res_lib_quorum_notification
The res_lib_quorum_notification struct.
Definition: ipc_quorum.h:80
internal_callback_pd::list
struct qb_list_head list
Definition: vsf_quorum.c:83
corodefs.h