1 /*-
2 * Copyright (c) 2003-2009 Silicon Graphics International Corp.
3 * Copyright (c) 2011 Spectra Logic Corporation
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions, and the following disclaimer,
11 * without modification.
12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
13 * substantially similar to the "NO WARRANTY" disclaimer below
14 * ("Disclaimer") and any redistribution must be conditioned upon
15 * including a substantially similar Disclaimer requirement for further
16 * binary redistribution.
17 *
18 * NO WARRANTY
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
28 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGES.
30 *
31 * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_ha.h#1 $
32 * $FreeBSD$
33 */
34
35 #ifndef _CTL_HA_H_
36 #define _CTL_HA_H_
37
38 /*
39 * CTL High Availability Modes:
40 *
41 * CTL_HA_MODE_ACT_STBY: One side is in Active state and processing commands,
42 * the other side is in Standby state, returning errors.
43 * CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write
44 * mirroring and read re-direction are assumed to
45 * happen in the back end.
46 * CTL_HA_MODE_XFER: Commands are serialized and data is transferred
47 * for write mirroring and read re-direction.
48 */
49
50 typedef enum {
51 CTL_HA_MODE_ACT_STBY,
52 CTL_HA_MODE_SER_ONLY,
53 CTL_HA_MODE_XFER
54 } ctl_ha_mode;
55
56
57 /*
58 * This is a stubbed out High Availability interface. It assumes two nodes
59 * staying in sync.
60 *
61 * The reason this interface is here, and stubbed out, is that CTL was
62 * originally written with support for Copan's (now SGI) high availability
63 * framework. That framework was not released by SGI, and would not have
64 * been generally applicable to FreeBSD anyway.
65 *
66 * The idea here is to show the kind of API that would need to be in place
67 * in a HA framework to work with CTL's HA hooks. This API is very close
68 * to the Copan/SGI API, so that the code using it could stay in place
69 * as-is.
70 *
71 * So, in summary, this is a shell without real substance, and much more
72 * work would be needed to actually make HA work. The implementation
73 * inside CTL will also need to change to fit the eventual implementation.
74 * The additional pieces we would need are:
75 *
76 * - HA "Supervisor" framework that can startup the components of the
77 * system, and initiate failover (i.e. active/active to single mode)
78 * and failback (single to active/active mode) state transitions.
79 * This framework would be able to recognize when an event happens
80 * that requires it to initiate state transitions in the components it
81 * manages.
82 *
83 * - HA communication framework. This framework should have the following
84 * features:
85 * - Separate channels for separate system components. The CTL
86 * instance on one node should communicate with the CTL instance
87 * on another node.
88 * - Short message passing. These messages would be fixed length, so
89 * they could be preallocated and easily passed between the nodes.
90 * i.e. conceptually like an ethernet packet.
91 * - DMA/large buffer capability. This would require some negotiation
92 * with the other node to define the destination. It could
93 * allow for "push" (i.e. initiated by the requesting node) DMA or
94 * "pull" (i.e. initiated by the target controller) DMA or both.
95 * - Communication channel status change notification.
96 * - HA capability in other portions of the storage stack. Having two CTL
97 * instances communicate is just one part of an overall HA solution.
98 * State needs to be synchronized at multiple levels of the system in
99 * order for failover to actually work. For instance, if CTL is using a
100 * file on a ZFS filesystem as its backing store, the ZFS array state
101 * should be synchronized with the other node, so that the other node
102 * can immediately take over if the node that is primary for a particular
103 * array fails.
104 */
105
106 /*
107 * Communication channel IDs for various system components. This is to
108 * make sure one CTL instance talks with another, one ZFS instance talks
109 * with another, etc.
110 */
111 typedef enum {
112 CTL_HA_CHAN_NONE,
113 CTL_HA_CHAN_CTL,
114 CTL_HA_CHAN_ZFS,
115 CTL_HA_CHAN_MAX
116 } ctl_ha_channel;
117
118 /*
119 * HA communication event notification. These are events generated by the
120 * HA communication subsystem.
121 *
122 * CTL_HA_EVT_MSG_RECV: Message received by the other node.
123 * CTL_HA_EVT_MSG_SENT: Message sent to the other node.
124 * CTL_HA_EVT_DISCONNECT: Communication channel disconnected.
125 * CTL_HA_EVT_DMA_SENT: DMA successfully sent to other node (push).
126 * CTL_HA_EVT_DMA_RECEIVED: DMA successfully received by other node (pull).
127 */
128 typedef enum {
129 CTL_HA_EVT_NONE,
130 CTL_HA_EVT_MSG_RECV,
131 CTL_HA_EVT_MSG_SENT,
132 CTL_HA_EVT_DISCONNECT,
133 CTL_HA_EVT_DMA_SENT,
134 CTL_HA_EVT_DMA_RECEIVED,
135 CTL_HA_EVT_MAX
136 } ctl_ha_event;
137
138 typedef enum {
139 CTL_HA_STATUS_WAIT,
140 CTL_HA_STATUS_SUCCESS,
141 CTL_HA_STATUS_ERROR,
142 CTL_HA_STATUS_INVALID,
143 CTL_HA_STATUS_DISCONNECT,
144 CTL_HA_STATUS_BUSY,
145 CTL_HA_STATUS_MAX
146 } ctl_ha_status;
147
148 typedef enum {
149 CTL_HA_DATA_CTL,
150 CTL_HA_DATA_ZFS,
151 CTL_HA_DATA_MAX
152 } ctl_ha_dtid;
153
154 typedef enum {
155 CTL_HA_DT_CMD_READ,
156 CTL_HA_DT_CMD_WRITE,
157 } ctl_ha_dt_cmd;
158
159 struct ctl_ha_dt_req;
160
161 typedef void (*ctl_ha_dt_cb)(struct ctl_ha_dt_req *);
162
163 struct ctl_ha_dt_req {
164 ctl_ha_dt_cmd command;
165 void *context;
166 ctl_ha_dt_cb callback;
167 ctl_ha_dtid id;
168 int ret;
169 uint32_t size;
170 uint8_t *local;
171 uint8_t *remote;
172 };
173
174 typedef void (*ctl_evt_handler)(ctl_ha_channel channel, ctl_ha_event event,
175 int param);
176 void ctl_ha_register_evthandler(ctl_ha_channel channel,
177 ctl_evt_handler handler);
178
179 static inline ctl_ha_status
ctl_ha_msg_create(ctl_ha_channel channel,ctl_evt_handler handler)180 ctl_ha_msg_create(ctl_ha_channel channel, ctl_evt_handler handler)
181 {
182 return (CTL_HA_STATUS_SUCCESS);
183 }
184
185 /*
186 * Receive a message of the specified size.
187 */
188 static inline ctl_ha_status
ctl_ha_msg_recv(ctl_ha_channel channel,void * buffer,unsigned int size,int wait)189 ctl_ha_msg_recv(ctl_ha_channel channel, void *buffer, unsigned int size,
190 int wait)
191 {
192 return (CTL_HA_STATUS_SUCCESS);
193 }
194
195 /*
196 * Send a message of the specified size.
197 */
198 static inline ctl_ha_status
ctl_ha_msg_send(ctl_ha_channel channel,void * buffer,unsigned int size,int wait)199 ctl_ha_msg_send(ctl_ha_channel channel, void *buffer, unsigned int size,
200 int wait)
201 {
202 return (CTL_HA_STATUS_SUCCESS);
203 }
204
205 /*
206 * Allocate a data transfer request structure.
207 */
208 static inline struct ctl_ha_dt_req *
ctl_dt_req_alloc(void)209 ctl_dt_req_alloc(void)
210 {
211 return (NULL);
212 }
213
214 /*
215 * Free a data transfer request structure.
216 */
217 static inline void
ctl_dt_req_free(struct ctl_ha_dt_req * req)218 ctl_dt_req_free(struct ctl_ha_dt_req *req)
219 {
220 return;
221 }
222
223 /*
224 * Issue a DMA request for a single buffer.
225 */
226 static inline ctl_ha_status
ctl_dt_single(struct ctl_ha_dt_req * req)227 ctl_dt_single(struct ctl_ha_dt_req *req)
228 {
229 return (CTL_HA_STATUS_WAIT);
230 }
231
232 /*
233 * SINGLE: One node
234 * HA: Two nodes (Active/Active implied)
235 * SLAVE/MASTER: The component can set these flags to indicate which side
236 * is in control. It has no effect on the HA framework.
237 */
238 typedef enum {
239 CTL_HA_STATE_UNKNOWN = 0x00,
240 CTL_HA_STATE_SINGLE = 0x01,
241 CTL_HA_STATE_HA = 0x02,
242 CTL_HA_STATE_MASK = 0x0F,
243 CTL_HA_STATE_SLAVE = 0x10,
244 CTL_HA_STATE_MASTER = 0x20
245 } ctl_ha_state;
246
247 typedef enum {
248 CTL_HA_COMP_STATUS_OK,
249 CTL_HA_COMP_STATUS_FAILED,
250 CTL_HA_COMP_STATUS_ERROR
251 } ctl_ha_comp_status;
252
253 struct ctl_ha_component;
254
255 typedef ctl_ha_comp_status (*ctl_hacmp_init_t)(struct ctl_ha_component *);
256 typedef ctl_ha_comp_status (*ctl_hacmp_start_t)(struct ctl_ha_component *,
257 ctl_ha_state);
258
259 struct ctl_ha_component {
260 char *name;
261 ctl_ha_state state;
262 ctl_ha_comp_status status;
263 ctl_hacmp_init_t init;
264 ctl_hacmp_start_t start;
265 ctl_hacmp_init_t quiesce;
266 };
267
268 #define CTL_HA_STATE_IS_SINGLE(state) ((state & CTL_HA_STATE_MASK) == \
269 CTL_HA_STATE_SINGLE)
270 #define CTL_HA_STATE_IS_HA(state) ((state & CTL_HA_STATE_MASK) == \
271 CTL_HA_STATE_HA)
272
273 #endif /* _CTL_HA_H_ */
274