diff --git a/COPYING-3_0.txt b/COPYING-3_0.txt new file mode 100644 index 0000000..729f5d7 --- /dev/null +++ b/COPYING-3_0.txt @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/HISTORY b/HISTORY new file mode 100644 index 0000000..069269d --- /dev/null +++ b/HISTORY @@ -0,0 +1,640 @@ + +Version 5.02 2014-06-23 + * corect README spell mistake + * bug fixed: can't deal sync truncate file exception + * remove tracker_global.c extern keyword to tracker_global.h + +Version 5.01 2014-02-02 + * trunk binlog be compressed when trunk init + * bug fixed: sync trunk binlog file to other storage servers immediately when + the trunk server init done + * move ioevent_loop.[hc] and fast_task_queue.[hc] from tracker/ to common/ + * hash table support locks + * hash talbe support new functions: hash_inc and hash_inc_ex + +Version 5.00 2013-12-23 + * discard libevent, use epoll in Linux, kqueue in FreeBSD, port in SunOS directly + * do_notify_leader_changed force close connection when target is myself + * modify the INSTALL file and tracker/Makefile.in + +Version 4.08 2013-11-30 + * bug fixed: FDFS_DOWNLOAD_SERVER_ROUND_ROBIN change to FDFS_STORE_SERVER_ROUND_ROBIN + * dio_init use memset to init buffer + * disable linger setting (setsockopt with option SO_LINGER) + * change log level from error to warning when file not exist on storage server + +Version 4.07 2013-06-02 + * make.sh add -lpthread by ldconfig check + * support multi accept threads + * tracker and storage server close client connection when recv invalid package + * client/storage_client.c: file_exist with silence flag + * tracker and storage process support start, stop and restart command + * tracker/tracker_proto.c fdfs_recv_header: logDebug change to logError + +Version 4.06 2013-01-24 + * fdfs_upload_file tool enhancement + * fdfs_download_file tool support offset and download size + * trunk file upload support sub paths rotating correctly + * add function: fdfs_http_get_file_extension + * sync truncate file operation anyway + +Version 4.05 2012-12-30 + * client/fdfs_upload_file.c can specify storage ip port and store path index + * add connection pool + * client load storage ids config + * common/ini_file_reader.c does NOT call chdir + * keep the mtime of file same + * use g_current_time instead of call time function + * remove embed HTTP support + +Version 4.04 2012-12-02 + * bug fixed: get storage server id when storage daemon init + * storage id in filename use global variable + * dynamic alloc memory 8 bytes alignment + * fast_task_queue support memory pool chain + +Version 4.03 2012-11-18 + * trunk_mgr/trunk_mem.c: log error and add more debug info + * file id generated by storage server can include storage server ID + +Version 4.02 2012-10-30 + * validate file_ext_name and prefix_name when upload file + * storage.conf add parameter: file_sync_skip_invalid_record + * add offset debug info when sync file fail + * bug fixed: log to binlog also if the file exists when sync file + * tracker and storage error log support rotate + * support rotate log by file size + * rotate log when receive HUP signal + * fdfs_monitor support set trunk server + * bug fixed: tracker_mem.c correct double mutex lock + +Version 4.01 2012-10-21 + * trunk_mgr/trunk_mem.c: trunk init flag check more strictly + * file signature for checking file duplicate support MD5 + * slave file support both symbol link and direct file + * tracker server log trunk server change logs + +Version 4.00 2012-10-06 + * identify storage server by ID instead of IP address + * tracker.conf: storage reserved space can use ratio such as 10% + * storage server support access log + * appender file and trunk file also use rand number in file id + * bug fixed: test_upload.c: char file_id[64] change to: char file_id[128] + * set pipe reading fd with attribute O_NOATIME + * bug fixed: correct php extension call_user_function TSRMLS_DC with TSRMLS_CC + +Version 3.11 2012-08-04 + * setsockopt set linger.l_linger to micro-seconds in FreeBSD and seconds + in others + * trunk binlog reader skip incorrect records + * bug fixed: single disk recovery support symbol link and trunk file + * storage generate filename enhancement + * ETIME change to ETIMEDOUT for FreeBSD + * tracker_mem.c: load storage server ignore empty ip address + +Version 3.10 2012-07-22 + * check and init trunk file more gracefully + * remove unused-but-set-variable + * bug fixed: return correct group name when g_check_file_duplicate is true + * bug fixed: php extension call_user_function replace TSRMLS_CC with TSRMLS_DC + * large the interval of tracker re-select trunk server + * trunk free block check duplicate using avl tree + * trunk file sync overwrite the dest file anyway + * common/avl_tree.c: free data when delete + * tracker.conf add parameter: trunk_init_reload_from_binlog, when this flag + is set to true, load all free trunk blocks from the trunk binlog + * trunk status control only by trunk_mem.c and memcmp struct FDFSTrunkFullInfo + avoid memory alignment problem + * auto remove the too old temp file + +Version 3.09 2012-07-08 + * make.sh avoid override config files of /etc/fdfs/ + * common/logger.c: function log_init can be called more than once + * php extension logInfo change to logDebug + * c client logInfo change to logDebug + * storage_dio.c log info more properly + * delete the trunk space which be occupied + * tracker.conf add parameter: trunk_init_check_occupying, when this flag + is set to true, do not add the trunk nodes which be occupied + * another method to get local ip addresses + +Version 3.08 2012-05-27 + * FAST_MAX_LOCAL_IP_ADDRS change from 4 to 16 + * appender file support modify + * appender file support truncate + +Version 3.07 2012-05-13 + * tracker/tracker_mem.c: check storage ip address is not empty + * remove direct IO support + * trunk binlog sync optimization + * php extension compile passed in PHP 5.4.0 + * get local ip addresses enhancement + * trunk server select the storage server whose binglog file size is max + * sync trunk binlog file correctly when trunk server changed + +Version 3.06 2012-01-22 + * add common/avl_tree.h and common/avl_tree.c + * organize trunk free blocks using AVL tree + * find the trunk server for each group when current tracker be a leader + * common/sched_thread.c can add schedule entry dynamicly + * support creating trunk file advancely + +Version 3.05 2011-12-20 + * remove compile warnings + * storage server's store_path_count can be more than that of group + * bug fixed: common/fast_mblock.c malloc bytes are not enough + * make.sh support OS: HP-UX + +Version 3.04 2011-11-25 + * bug fixed: duplicate files only save one entry ok with trunk file mode + * bug fixed: sync correctly with more binlog files + * fdfs_file_info query file info from storage server + * bug fixed: php extension compile error using gcc 4.6.1 as: + variable 'store_path_index' set but not used + * bug fixed: delete the metadata of trunked file correctly + * bug fixed: append file ok when check duplicate is on + * storage/trunk_mgr/trunk_shared.[hc]: trunk_file_stat_func do not + use function pointer + * bug fixed: storage/trunk_mgr/trunk_shared.c base64_decode_auto + overflow 1 byte + * bug fixed: delete slave file correctly + * bug fixed: remove debug info + * md5 function name changed to avoid conflict + +Version 3.03 2011-10-16 + * ignore existed link when sync link file + * http token checking support persistent token + * add functions: storage_file_exist and storage_file_exist1 + * php minfo add fastdfs version info + * make.sh changed + * client move libevent dependency + +Version 3.02 2011-09-18 + * bug fixed: tracker_mem_check_add_tracker_servers add tracker server + correctly + * php client compile ok with php 5.2.17 + * re-select trunk server ok + +Version 3.01 2011-07-31 + * bug fixed: tracker_get_connection_ex and tracker_get_connection_r_ex + connect two times with multi tracker servers + * bug fixed: tracker_mem_check_add_tracker_servers condition not correct + * all logError add source filename and line + * php extension support upload file callback + * php extension support download file callback + +Version 3.00 2011-06-19 + * mass small files optimization + * add fixed block memory pool: common/fast_mblock.c + * bug fixed: tracker_mem.c do NOT clear g_groups fields + * bug fixed: slave file and appender file download ok + * bug fixed: tracker / storage run by group / user, set file owner + * tracker server support leader + * client support static library + * client_func.h add functions fdfs_tracker_group_equals and + fdfs_get_file_ext_name + * bug fixed: test/dfs_func_pc.c compile ok + * storage server check free space enough when upload a file + +Version 2.09 2011-02-19 + * bug fixed: write_to_binlog_index then increase g_binlog_index (feedback + by koolcoy) + * disk read / write supports direct mode (avoid caching by the file system) + +Version 2.08 2011-01-30 + * bug fixed: fdfs_trackerd.c set g_tracker_thread_count to 0 + * add cmd TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP to support list one group + * support disk recovery automatically + * support total_upload_bytes, success_upload_bytes, total_download_bytes and + success_download_bytes etc. 18 stat fields + * tracker data file storage_groups.dat changes to storage_groups_new.dat, and + storage_servers.dat changes to storage_servers_new.dat + * support file append, add tests: fdfs_appender_test and fdfs_appender_test1 + * storage_dio.c: dio_deal_task split to several functions + * tracker http check thread exit normally + * function fdfs_get_file_info_ex changed, add function fdfs_get_file_info_ex1 + * fix some type cast error when compile with c++ + * client add tools: fdfs_upload_appender and fdfs_append_file + +Version 2.07 2011-01-09 + * slave file's prefix name can be empty + * FDFS_MAX_GROUPS change from 64 to 512 + * file size field in the file id changed: high 32 bits is random integer + when the file size < 2GB and the highest bit set to 1 + * tracker_service.c: in function list_group_storages, use strcpy + intead of memcpy + * php extension add function fastdfs_tracker_delete_storage + * client add tool: fdfs_file_info to get file info, including file size, + create timestamp, source storage ip address and crc32 signature + * fdfs_upload_file.c: omit more error info when the local file not exist + +Version 2.06 2010-12-26 + * sync file op: do not sync the file which exists on dest storage server + and the file size are same + * bug fixed: sync copy file will clear the existed file on dest storage + server (truncate the file size to 0), this bug caused by V2.04 + * bug fixed: make temp file discard system function mkstemp, + use file sequence No. with pthread_mutex_lock + * bug fixed: function fastdfs_tracker_list_groups, when parameter group_name + is null or empty string, return all groups info + * bug fixed: upload a file extends 2GB will fail + * bug fixed: tracker to tracker sync system data files, in function: + tracker_mem_get_tracker_server, pTrackerStatus not be set properly + +Version 2.05 2010-12-05 + * client/fdfs_monitor.c: add sync delay time + * tracker/fast_task_queue.c: pTask->data = pTask->arg + arg_size; + change to: pTask->data = (char *)pTask->arg + arg_size; + * bug fixed: storage_sync.c line 237 cause core dump in Ubuntu 10.04 + * upload file test use mmap, support more test_upload processes + * client add three tools: fdfs_upload_file, fdfs_download_file and + fdfs_delete_file + +Version 2.04 2010-11-19 + * storage.conf: tracker server ip can NOT be 127.0.0.1 + * do not catch signal SIGABRT + * strerror change to STRERROR macro + * sync copy file use temp filename first, rename to the correct filename + when sync done + * file id use 4 bytes CRC32 signature instead of random number + * add file: client/fdfs_crc32.c + * one of file hash code signature function change from APHash_ex + to simple_hash_ex + * bug fixed: when fdfs_storaged quit, maybe write to binlog file fail, + the error info is "Bad file descriptor" + +Version 2.03 2010-11-08 + * bug fixed: core dump when http.need_find_content_type=false and + http.anti_steal.check_token=true + * storage server add join_time field (create timestamp of this storage) + * tracker server fetch system files from other tracker server when + first storage server join in (tracker to tracker sync system files) + * tracker server changes the old ip address to the new address when the + storage server ip address changed + * tracker to tracker sync system data files in some case, multi tracker + server supported well + +Version 2.02 2010-10-28 + * get parameters function from tracker server changed, + add paramter: storage_sync_file_max_delay + * local ip functions move to common/local_ip_func.c + * when query all storage servers to store, do not increase the current + write server index + * struct FDFSHTTPParams add field: need_find_content_type + * symbol link client library to /usr/lib64 in 64 bits OS + * storage_client.c: deal file extension name correctly + +Version 2.01 2010-10-17 + * client/fdfs_monitor.c can specify tracker server + * micro STORAGE_STORE_PATH_PREFIX_CHAR change to + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR + * php extension can set log filename + * php extension add function: fastdfs_client_version + * bug fixed: client/tracker_client.c tracker_get_connection_ex NULL pointer + * set max core dump file size to at least 256MB when DEBUG_FLAG is on, + make sure to generate core file when core dump with DEBUG_FLAG on + * upload file can get available storage server list of the group, + add command TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL and + TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL + * bug fixed: storage core dump in some case + +Version 2.00 2010-08-22 + * tracker network io use libevent instead of traditional io model + * storage network io use libevent instead of traditional io model + * storage disk read/write use separate threads + * tracker_mem.c malloc single group and storage struct, remove referer + * make install copy config files + * tracker.conf add two parameters: storage_sync_file_max_delay and + storage_sync_file_max_time + * client tracker_get_connection increase server_index correctly + * storage sync to storage server adds active test + * test programs compile ok + +Version 1.29 2010-06-30 + * add files: tracker_dump.h and tracker_dump.c, tracker dump global vars + * add files: storage_dump.h and storage_dump.c, storage dump global vars + * sockopt.c: tcprecvfile and tcpdiscard add parameter total_recv_bytes + * storage server add fields: storage_port and storage_http_port + * auto rename synced remark files when the port of all storage servers + in a group changed to another port + * connect server support timeout, adding connect_timeout parameter in + config file + * log_init set log to cache to false (no cache) + +Version 1.28 2010-05-30 + * tracker_servive.c: set current_write_group anyway when current group + out of space + * logger support context (multi instance) + * get storage servers by filename: if the file created one day ago (the create + timestamp of the file < current_time - 86400), any active storage server matches + * add files: common/pthread_func.h and common/pthread_func.c + * common/sched_thread.h, remove statement: extern bool g_continue_flag; + * client add libfastcommon + * global variables: g_base_path, g_network_timeout, g_version change to + g_fdfs_base_path, g_fdfs_network_timeout, g_fdfs_version + * common/fdfs_base64.h/c change name to common/base64.h/c + * make.sh use TARGET_PREFIX instead of TARGET_PATH + * protocol add ACTIVE_TEST, tracker and storage both support + * php client, bug fixed: fastdfs_connect_server, the sock must init to -1 + * bug fixed: storage status not correct with multi tracker servers + * sync storage mark file and stat file to disk properly + +Version 1.27 2010-04-10 + * storage.conf: add if_alias_prefix parameter to get the ip address of the + local host + * storage http support domain name + * php extension add some parameters in fastdfs_client.ini + * make.sh compile use debug mode + * type off_t change to int64_t + * redirect stdout and stderr to log file + * php extension list_groups add fields: version and http_domain + +Version 1.26 2010-02-28 + * remove compile warning of logError + * ini reader support section + * bug fixed: tracker/tracker_mem.c sync storage server status + * use storage server http server port anyway + * bug fixed: ini reader can support relative config filename + * function enhancement: tracker server can check storage HTTP server alive + +Version 1.25 2010-02-04 + * storage_sync.c if source file not exist when sync a file, change from + logWarning to logDebug + * filename buff size change from 64 to 128 + * bug fixed: c client and php client, log not inited cause core dump when + call log functions + * can print stack trace when process core dumped in Linux server + * bug fixed: tracker/tracker_mem.c load storage servers fail with many groups + and storage servers + * common/sockopt.c remove debug info + * storage stat add fields: version + * auto adjust when storage server ip address changed + * bug fixed: when add a new storage server, other storage servers' status keep + the same, not changed + * add macros, compile passed in cygwin, thanks Seapeak + * write to system data file using lock + * common/ini_file_reader.c: use one context parameter, not two parameters + * storage status sync modified (the code of tracker and storage both changed) + * when recv kill signal, worker thread quit more quickly, daemon process + fdfs_trackerd and fdfs_storage quit very quickly when recv kill signal + * remove compile warning info of logError + * tracker server start more quickly with many groups and storage servers + * bug fixed: correct off_t printf format + +Version 1.24 2010-01-06 + * call php_fdfs_close with TSRMLS_CC as php_fdfs_close(i_obj TSRMLS_CC) + * storage server to storage server report ip address as tracker client + * bug fixed: sendfile exceeds 2GB file in Linux + * bug fixed: delete storage server + * storage stat add fields: up_time and src_ip_addr + * big static or struct memeber char array buffer change to malloc in order to + decrease stack size + * FDFS_WRITE_BUFF_SIZE change from 512KB to 256KB + * bug fixed: client/storage_client.c, meta data miss when upload file + * decrease thread_stack_size default value in config files: tracker.conf + and storage.conf + +Version 1.23 2009-11-29 + * remove unuseless variable "sleep_secs" in tracker_report_thread_entrance + * storage can bind an address when connect to other servers (as a client) + * common/md5.h fix UINT4 typedef wrong type in 64 bit OS + * client/fdfs_test.c: print the source ip address decoded from the remote + filename + * client add function fdfs_get_file_info + * php extension add functions: fastdfs_http_gen_token and fastdfs_get_file_info + * server process will exit when the http service starts fail + * support file group, a master file with many slave files whose file id can be + combined from master file id and prefix + * php client support uploading slave file + * ip address in filename change from host byte order to network byte order + * storage sync performance enhancement, using read buffer of 64KB to avoid + reading binlog file repeatly + * storage add prototol cmd: STORAGE_PROTO_CMD_QUERY_FILE_INFO + * FDFS_FILE_EXT_NAME_MAX_LEN changed from 5 to 6 + * get file info support slave file + * storage server for uploading file support priority + +Version 1.22 2009-10-12 + * bug fixed: common/shared_func.c allow_hosts array maybe overflow in some case + * tracker/tracker_mem.c: protocol TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL, + return at least a storage server when active storage + server count of the group > 0 + * bug fixed: when client connection disconnected, always log debug or error info + * make.sh: default not install FastDFS services in Linux server + * common/sockopt.c: setsockopt level SOL_TCP only supported in Linux + * common/http_func.c: do not use function strsep because strsep is not portable + * client upload file support callback function + * client support multi tracker groups (multi FastDFS clusters) + * bug fixed: thread_stack_size not correct when the param thread_stack_size + not set in the config file + * supply php extension (directory name: php_client) + * c client reconnect server (tracker or storage) when network IO error + * c client: make tracker server index counter thread safely + +Version 1.21 2009-09-19 + * bug fixed: when source storage server synced file to new storage server done, + it's status changed to ONLINE (should keep as ACTIVE, report by zhouzezhong) + * add thread_stack_size in config file, default value is 1MB (report by chhxo) + * tracker and storage server use setsockopt to keep alive + (report by zhouzezhong) + * bug fixed: storage server with multi-path, upload file fail when the free + space of each path <= reserved space (the total free space > reserved space, + report by zhouzezhong) + * storage_sync.c: when connect fail, do not change the dest storage server ' + status to offline + * tracker_service.c and storage_service.c change log level from WARNING to DEBUG + when client connection disconnected (report by Jney402) + * bug fixed: tracker_client.c correct store_path_index return by tracker server + (report by happy_fastdfs) + * bug fixed: tracker_service.c when store_lookup set to 2 (load balance), use + another pthread lock to avoid long time lock waiting + (report by happy_fastdfs) + * add service shell scripts in directory: init.d + (services will auto installed on Linux, report by hugwww) + +Version 1.20 2009-09-05 + * base64 use context, functions changed + * common/ini_file_reader.c: fix memory leak + * tracker server support HTTP protocol, one thread mode + * storage server support HTTP protocol, one thread mode + * fix bug: storage server rebuild, auto sync data correctly + * fix bug: sync data fail (correct storage server status) + * when storage server idle time exceeds check_active_interval seconds, + set it's status to offline + * tracker counter thread safely + +Version 1.19 2009-07-23 + * use poll instead of select in sockopt.c + * hash.c use chain impl by self + * use FastDHT 1.09 client code + * ini reader support HTTP protocol, conf file can be an url + * correct test dir compile error + * use non-block socket to increase network IO performance + * add cmd TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL: query all storage servers + from which the file can be dowloaded + * while (1) ... break; changed to do ... while (0); + +Version 1.18 2009-05-24 + * restart.sh only kill the programs match the program name and all parameters + * correct get local ip addresses + * common files do not use global vars like g_network_timeout and g_base_path + * download file support offset and download bytes + * hash function change type from unsigned int to signed int + * file size in file name support 64 bits, old bytes is 4, new bytes is 8 + +Version 1.17 2009-03-19 + * add test programs at sub directory test/ + * common/shared_func.c: rindex change to strrchr, add #include + * support SunOS (Solaris), compile passed on SunOS 5.10 + * support AIX, compile passed on AIX 5.3 + * sys call statfs change to statvfs + * use scheduling thread to sync binlog buff / cache to disk, add parameter + "sync_binlog_buff_interval" to conf file storage.conf + * use FastDHT v1.07 client code + +Version 1.16 2009-02-14 + * client can specify group name when upload file + * tracker_service.c: cmd dispatch changed to "switch ... case" + not "if ... else if" + * storage_service.c: call fdfs_quit before tracker_disconnect_server + +Version 1.15 2009-01-28 + * use FastDHT v1.04 client code + * use FastDHT client thread safely + +Version 1.14 2009-01-18 + * storage/storage_sync.c: + old: if (reader.sync_row_count % 1000 == 0) + new: if (reader.scan_row_count % 2000 == 0) + * little adjustment for common files can be used by FastDHT + * sched_thread.h /.c add global variable g_schedule_flag to quit normally + * shared_func.h / .c add function get_time_item_from_conf + * sched_thread.h /.c support time_base of task + * hash.h / .c add function CRC32, add hash function to support stream hash + * add FastDHT client files in storage/fdht_client/ + * create symbol link when the file content is duplicate, + add item "check_file_duplicate" to conf file storage.conf + * use FastDHT v1.02 client code + * auto delete invalid entry in FastDHT when the source file does not exist + +Version 1.13 2008-11-29 + * re-calculate group 's free space when one of it's storage servers' + free space increase + * add parameters: sync_interval, sync_start_time and sync_end_time to + storage.conf + * performance enhancement: log to buffer, flush to disk every interval seconds + * standard fds closed by daemon_init: 0(stdin), 1(stdout) and 2(stderr) + * fix bug: pthread_kill sometimes cause core dump when program terminated + * fix bug: sync.c open next binlog cause loop call + +Version 1.12 2008-11-12 + * storage server support multi path (mount point) + * upload file support file ext name, add source storage ip address to filename + * add delete command to delete the invalid storage server + * add client functions which combine group name and filename to file id, + add anothor client test program: fdfs_test1.c to use file id + * client download file support callback function + * add protocol cmd TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE, + and client API add tracker_query_storage_update + * add protocol cmd TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT to report last + synced timestamp as dest server + * fix sync old data files to new server bug + * fcntl change to pthread_mutex_lock + +Version 1.11 2008-10-04 + * kill report and sync threads when recv terminate signal + * add item "store_server" in tracker.conf, by default use the first + storage server to store uploaded files + * ini_file_reader.c changed: a conf file can include other conf files + * some adjustment: + some macro name changed + add common_define.h + remove fdfs_define.c + fdfs_os_bits.h change to _os_bits.h + +Version 1.10 2008-09-20 + * performance optimizing: use thread pool, create all work threads at startup + * trim function op in shared_func.c + * add Makefile template Makefile.in, delete Makefile and Makefile.freebsd + change make.sh to support all unix systems (passed in Linux and FreeBSD) + +Version 1.9 2008-09-14 + * security enhancement: support allow hosts which can connect to the server + * server can be run by the specified group and user, set by the config file + * change make.sh and add file common/fdfs_os_bits.h, + remove the warning info of printf format for int64_t param in 64 bits system + * storage_client.c changed: auto connect to storage server when not connected + * change some macro name and function name in tracker/tracker_proto.h + +Version 1.8 2008-09-07 + * communication protocol changed to support large file exceed 2GB: + # all integer field is 8 bytes big-endian + # group name fixed length: FDFS_GROUP_NAME_MAX_LEN bytes + * storage stat numbers (such as total_upload_count, success_upload_count) + use int64_t (8 bytes integer) + * ini_file_reader.c add function iniGetInt64Value + * sockopt.c add function tcpsetnonblockopt + * shared_func.c add function set_nonblock + +Version 1.7 2008-08-31 + * performance optimizing: + # change fopen to syscall open + # increase the efficiency of socket functions tcpsenddata and tcprecvdata + * change the return value of socket funtions such as tcpsenddata, + tcprecvdata and connectserverbyip + old return value: result=1 for success, result != 1 fail + new return value: result=0 for success, result != 0 fail, return the error code + * log function enhancement: + # support log level + # parameter "log_level" added to server config file + # keep the log file opened to increase performance + * fix log format and parameter mismatched bug (check by printf) + * log CRIT message to log file when program exit unexpectedly + * Makefile add compile flag -D_FILE_OFFSET_BITS=64 to support large files + * change the type of file_size and file_offset to off_t + * change signal to sigaction + * fix client Makefile to compile library correctly + * restart.sh modified: use external command "expr" to replace shell command "let" + +Version 1.6 2008-08-24 + * add restart daemon shell script: restart.sh + * use setrlimit to increase max open files if necessary + * security enhancement: the format of data filename must be: HH/HH/filename, + eg. B9/F4/SLI2NAAMRPR9r8.d + * fix bug: errno is not correct where the downloaded file does not exist, + communication is broken when the download file is a directory + +Version 1.5 2008-08-17 + * add client function storage_download_file_to_file + * use pthread_attr_setstacksize to increase thread stack size to 1 MB + * use sendfile syscall to send file in Linux and FreeBSD + * fix bug: add O_TRUNC flag when open file to write + * remove warning info compiled by gcc 4.2 + * fcntl set lock.l_len to 0 + +Version 1.4 2008-08-10 + * storage server recv file method change + old method: recv the whole file content/buff before write to file + new method: write to file once recv a certain bytes file buff, eg. 128KB buff size + * storage client and storage server send file method change + old method: get the whole file content/buff, then send to storage server + new method: send file to storage server more times. get a certain bytes file buff, then send to storage server + * upload file package remove the one pad byte field + * remove storage status FDFS_STORAGE_STATUS_DEACTIVE and add FDFS_STORAGE_STATUS_DELETED + +Version 1.3 2008-08-03 + * fix bug: when meta data is empty, get meta data return error + * support java client + # memset response header to 0 + # add group_name to upload file response package + +Version 1.2 2008-07-27 + * add client function storage_set_metadata to support setting metadata(overwrite or merge) + +Version 1.1 2008-07-20 + * implement storage disk report + * storing load balance between storage groups(volumes) when set store_lookup to 2 + +Version 1.0 2008-07-12 + * first version + diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..578da81 --- /dev/null +++ b/INSTALL @@ -0,0 +1,188 @@ +Copy right 2009 Happy Fish / YuQing + +FastDFS may be copied only under the terms of the GNU General +Public License V3, which may be found in the FastDFS source kit. +Please visit the FastDFS Home Page for more detail. +English language: http://english.csource.org/ +Chinese language: http://www.csource.org/ + +#step 1. download FastDFS source package and unpack it, +tar xzf FastDFS_v5.x.tar.gz +#for example: +tar xzf FastDFS_v5.01.tar.gz + +#step 2. enter the FastDFS dir +cd FastDFS + +#step 3. execute: +./make.sh + +#step 4. make install +./make.sh install + +#step 5. edit/modify the config file of tracker and storage + +#step 6. run server programs +#start the tracker server: +/usr/local/bin/fdfs_trackerd /etc/fdfs/tracker.conf restart +#in Linux, you can start fdfs_trackerd as a service: +/sbin/service fdfs_trackerd start + +#start the storage server: +/usr/local/bin/fdfs_storaged /etc/fdfs/storage.conf restart +#in Linux, you can start fdfs_storaged as a service: +/sbin/service fdfs_storaged start + +#step 7. run test program +#run the client test program: +/usr/local/bin/fdfs_test +/usr/local/bin/fdfs_test1 +#for example, upload a file: +/usr/local/bin/fdfs_test conf/client.conf upload /usr/include/stdlib.h + +#step 8. run monitor program +#run the monitor program: +/usr/local/bin/fdfs_monitor + + +tracker server config file sample please see conf/tracker.conf + +storage server config file sample please see conf/storage.conf + +client config file sample please see conf/client.conf + + +Item detail +1. server common items +--------------------------------------------------- +| item name | type | default | Must | +--------------------------------------------------- +| base_path | string | | Y | +--------------------------------------------------- +| disabled | boolean| false | N | +--------------------------------------------------- +| bind_addr | string | | N | +--------------------------------------------------- +| network_timeout | int | 30(s) | N | +--------------------------------------------------- +| max_connections | int | 256 | N | +--------------------------------------------------- +| log_level | string | info | N | +--------------------------------------------------- +| run_by_group | string | | N | +--------------------------------------------------- +| run_by_user | string | | N | +--------------------------------------------------- +| allow_hosts | string | * | N | +--------------------------------------------------- +| sync_log_buff_interval| int | 10(s) | N | +--------------------------------------------------- +| thread_stack_size | string | 1M | N | +--------------------------------------------------- +memo: + * base_path is the base path of sub dirs: + data and logs. base_path must exist and it's sub dirs will + be automatically created if not exist. + $base_path/data: store data files + $base_path/logs: store log files + * log_level is the standard log level as syslog, case insensitive + # emerg: for emergency + # alert + # crit: for critical + # error + # warn: for warning + # notice + # info + # debug + * allow_hosts can ocur more than once, host can be hostname or ip address, + "*" means match all ip addresses, can use range like this: 10.0.1.[1-15,20] + or host[01-08,20-25].domain.com, for example: + allow_hosts=10.0.1.[1-15,20] + allow_hosts=host[01-08,20-25].domain.com + +2. tracker server items +--------------------------------------------------- +| item name | type | default | Must | +--------------------------------------------------- +| port | int | 22000 | N | +--------------------------------------------------- +| store_lookup | int | 0 | N | +--------------------------------------------------- +| store_group | string | | N | +--------------------------------------------------- +| store_server | int | 0 | N | +--------------------------------------------------- +| store_path | int | 0 | N | +--------------------------------------------------- +| download_server | int | 0 | N | +--------------------------------------------------- +| reserved_storage_space| string | 1GB | N | +--------------------------------------------------- + +memo: + * the value of store_lookup is: + 0: round robin (default) + 1: specify group + 2: load balance (supported since V1.1) + * store_group is the name of group to store files. + when store_lookup set to 1(specify group), + store_group must be set to a specified group name. + * reserved_storage_space is the reserved storage space for system + or other applications. if the free(available) space of any stoarge + server in a group <= reserved_storage_space, no file can be uploaded + to this group (since V1.1) + bytes unit can be one of follows: + # G or g for gigabyte(GB) + # M or m for megabyte(MB) + # K or k for kilobyte(KB) + # no unit for byte(B) + +3. storage server items +------------------------------------------------- +| item name | type | default | Must | +------------------------------------------------- +| group_name | string | | Y | +------------------------------------------------- +| tracker_server | string | | Y | +------------------------------------------------- +| port | int | 23000 | N | +------------------------------------------------- +| heart_beat_interval | int | 30(s) | N | +------------------------------------------------- +| stat_report_interval| int | 300(s) | N | +------------------------------------------------- +| sync_wait_msec | int | 100(ms) | N | +------------------------------------------------- +| sync_interval | int | 0(ms) | N | +------------------------------------------------- +| sync_start_time | string | 00:00 | N | +------------------------------------------------- +| sync_end_time | string | 23:59 | N | +------------------------------------------------- +| store_path_count | int | 1 | N | +------------------------------------------------- +| store_path0 | string |base_path| N | +------------------------------------------------- +| store_path# | string | | N | +------------------------------------------------- +|subdir_count_per_path| int | 256 | N | +------------------------------------------------- +|check_file_duplicate | boolean| 0 | N | +------------------------------------------------- +| key_namespace | string | | N | +------------------------------------------------- +| keep_alive | boolean| 0 | N | +------------------------------------------------- +| sync_binlog_buff_interval| int | 60s | N | +------------------------------------------------- + +memo: + * tracker_server can ocur more than once, and tracker_server format is + "host:port", host can be hostname or ip address. + * store_path#, # for digital, based 0 + * check_file_duplicate: when set to true, must work with FastDHT server, + more detail please see INSTALL of FastDHT. FastDHT download page: + http://code.google.com/p/fastdht/downloads/list + * key_namespace: FastDHT key namespace, can't be empty when + check_file_duplicate is true. the key namespace should short as possible + diff --git a/README.md b/README.md index 8b6e2da..5aa4cee 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,44 @@ -fastdfs -======= +Copyright (C) 2008 Happy Fish / YuQing -FastDFS is an open source high performance distributed file system (DFS). It's major functions include: file storing, file syncing and file accessing, and design for high capacity and load balance. +FastDFS may be copied only under the terms of the GNU General +Public License V3, which may be found in the FastDFS source kit. +Please visit the FastDFS Home Page for more detail. +English language: http://english.csource.org/ +Chinese language: http://www.csource.org/ + + +FastDFS is an open source high performance distributed file system. It's major +functions include: file storing, file syncing and file accessing (file uploading +and file downloading), and it can resolve the high capacity and load balancing +problem. FastDFS should meet the requirement of the website whose service based +on files such as photo sharing site and video sharing site. + +FastDFS has two roles: tracker and storage. The tracker takes charge of +scheduling and load balancing for file access. The storage store files and it's +function is file management including: file storing, file syncing, providing file +access interface. It also manage the meta data which are attributes representing +as key value pair of the file. For example: width=1024, the key is "width" and +the value is "1024". + +The tracker and storage contain one or more servers. The servers in the tracker +or storage cluster can be added to or removed from the cluster by any time without +affecting the online services. The servers in the tracker cluster are peer to peer. + +The storarge servers organizing by the file volume/group to obtain high capacity. +The storage system contains one or more volumes whose files are independent among +these volumes. The capacity of the whole storage system equals to the sum of all +volumes' capacity. A file volume contains one or more storage servers whose files +are same among these servers. The servers in a file volume backup each other, +and all these servers are load balancing. When adding a storage server to a +volume, files already existing in this volume are replicated to this new server +automatically, and when this replication done, system will switch this server +online to providing storage services. + +When the whole storage capacity is insufficiency, you can add one or more +volumes to expand the storage capacity. To do this, you need to add one or +more storage servers. + +The identification of a file is composed of two parts: the volume name and +the file name. + +Client test code use client library please refer to the directory: client/test. diff --git a/client/Makefile.in b/client/Makefile.in new file mode 100644 index 0000000..a27abb8 --- /dev/null +++ b/client/Makefile.in @@ -0,0 +1,115 @@ +.SUFFIXES: .c .o .lo + +COMPILE = $(CC) $(CFLAGS) +ENABLE_STATIC_LIB = $(ENABLE_STATIC_LIB) +ENABLE_SHARED_LIB = $(ENABLE_SHARED_LIB) +INC_PATH = -I../common -I../tracker -I/usr/local/include +LIB_PATH = $(LIBS) +TARGET_PATH = $(TARGET_PREFIX)/bin +TARGET_LIB = $(TARGET_PREFIX)/lib +TARGET_INC = $(TARGET_PREFIX)/include +CONFIG_PATH = $(TARGET_CONF_PATH) + +FAST_STATIC_OBJS = ../common/hash.o ../common/chain.o \ + ../common/shared_func.o ../common/ini_file_reader.o \ + ../common/logger.o ../common/sockopt.o \ + ../common/base64.o ../common/sched_thread.o \ + ../common/http_func.o ../common/md5.o \ + ../common/pthread_func.o ../common/local_ip_func.o \ + ../common/avl_tree.o ../common/connection_pool.o + +FDFS_STATIC_OBJS = ../common/fdfs_global.o ../common/fdfs_http_shared.o \ + ../common/mime_file_parser.o ../tracker/tracker_proto.o \ + ../tracker/fdfs_shared_func.o \ + ../storage/trunk_mgr/trunk_shared.o \ + tracker_client.o client_func.o \ + client_global.o storage_client.o + +STATIC_OBJS = $(FAST_STATIC_OBJS) $(FDFS_STATIC_OBJS) + +FAST_SHARED_OBJS = ../common/hash.lo ../common/chain.lo \ + ../common/shared_func.lo ../common/ini_file_reader.lo \ + ../common/logger.lo ../common/sockopt.lo \ + ../common/base64.lo ../common/sched_thread.lo \ + ../common/http_func.lo ../common/md5.lo \ + ../common/pthread_func.lo ../common/local_ip_func.lo \ + ../common/avl_tree.lo ../common/connection_pool.lo + +FDFS_SHARED_OBJS = ../common/fdfs_global.lo ../common/fdfs_http_shared.lo \ + ../common/mime_file_parser.lo ../tracker/tracker_proto.lo \ + ../tracker/fdfs_shared_func.lo \ + ../storage/trunk_mgr/trunk_shared.lo \ + tracker_client.lo client_func.lo \ + client_global.lo storage_client.lo + +FAST_HEADER_FILES = ../common/common_define.h ../common/hash.h \ + ../common/chain.h ../common/logger.h \ + ../common/base64.h ../common/shared_func.h \ + ../common/pthread_func.h ../common/ini_file_reader.h \ + ../common/sockopt.h ../common/sched_thread.h \ + ../common/http_func.h ../common/md5.h ../common/_os_bits.h \ + ../common/local_ip_func.h ../common/avl_tree.h \ + ../common/connection_pool.h + +FDFS_HEADER_FILES = ../common/fdfs_define.h ../common/fdfs_global.h \ + ../common/mime_file_parser.h ../common/fdfs_http_shared.h \ + ../tracker/tracker_types.h ../tracker/tracker_proto.h \ + ../tracker/fdfs_shared_func.h \ + ../storage/trunk_mgr/trunk_shared.h \ + tracker_client.h storage_client.h storage_client1.h \ + client_func.h client_global.h fdfs_client.h + +ALL_OBJS = $(STATIC_OBJS) $(FAST_SHARED_OBJS) $(FDFS_SHARED_OBJS) + +ALL_PRGS = fdfs_monitor fdfs_test fdfs_test1 fdfs_crc32 fdfs_upload_file \ + fdfs_download_file fdfs_delete_file fdfs_file_info \ + fdfs_appender_test fdfs_appender_test1 fdfs_append_file \ + fdfs_upload_appender + +STATIC_LIBS = libfastcommon.a libfdfsclient.a + +SHARED_LIBS = libfastcommon.so.1 libfdfsclient.so.1 + +ALL_LIBS = $(STATIC_LIBS) $(SHARED_LIBS) + +all: $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS) +libfastcommon.so.1: + $(COMPILE) -o $@ $< -shared $(FAST_SHARED_OBJS) $(LIB_PATH) + ln -fs libfastcommon.so.1 libfastcommon.so +libfdfsclient.so.1: + $(COMPILE) -o $@ $< -shared $(FDFS_SHARED_OBJS) $(LIB_PATH) -L. -lfastcommon + ln -fs libfdfsclient.so.1 libfdfsclient.so +libfastcommon.a: + ar cru $@ $< $(FAST_STATIC_OBJS) +libfdfsclient.a: + ar cru $@ $< $(FDFS_STATIC_OBJS) +.o: + $(COMPILE) -o $@ $< $(STATIC_OBJS) $(LIB_PATH) $(INC_PATH) +.c: + $(COMPILE) -o $@ $< $(STATIC_OBJS) $(LIB_PATH) $(INC_PATH) +.c.o: + $(COMPILE) -c -o $@ $< $(INC_PATH) +.c.lo: + $(COMPILE) -c -fPIC -o $@ $< $(INC_PATH) +install: + mkdir -p $(TARGET_PATH) + mkdir -p $(CONFIG_PATH) + mkdir -p $(TARGET_LIB) + cp -f $(ALL_PRGS) $(TARGET_PATH) + if [ $(ENABLE_STATIC_LIB) -eq 1 ]; then cp -f $(STATIC_LIBS) $(TARGET_LIB); fi + if [ $(ENABLE_SHARED_LIB) -eq 1 ]; then cp -f $(SHARED_LIBS) $(TARGET_LIB); fi + if [ $(ENABLE_SHARED_LIB) -eq 1 ]; then ln -fs $(TARGET_LIB)/libfastcommon.so.1 $(TARGET_LIB)/libfastcommon.so; fi + if [ $(ENABLE_SHARED_LIB) -eq 1 ]; then ln -fs $(TARGET_LIB)/libfdfsclient.so.1 $(TARGET_LIB)/libfdfsclient.so; fi + + if [ ! -f $(CONFIG_PATH)/client.conf ]; then cp -f ../conf/client.conf ../conf/http.conf $(CONFIG_PATH); fi + + mkdir -p $(TARGET_INC) + mkdir -p $(TARGET_INC)/fastcommon + mkdir -p $(TARGET_INC)/fastdfs + cp -f $(FAST_HEADER_FILES) $(TARGET_INC)/fastcommon + cp -f $(FDFS_HEADER_FILES) $(TARGET_INC)/fastdfs + + if [ $(ENABLE_SHARED_LIB) -eq 1 -a $(TARGET_LIB) = "/usr/local/lib" ]; then sh ./fdfs_link_library.sh; fi +clean: + rm -f $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS) libfastcommon.so libfdfsclient.so + diff --git a/client/client_func.c b/client/client_func.c new file mode 100644 index 0000000..c1d7c49 --- /dev/null +++ b/client/client_func.c @@ -0,0 +1,537 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//client_func.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "base64.h" +#include "sockopt.h" +#include "shared_func.h" +#include "ini_file_reader.h" +#include "connection_pool.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "client_global.h" +#include "client_func.h" + +static int storage_cmp_by_ip_and_port(const void *p1, const void *p2) +{ + int res; + + res = strcmp(((ConnectionInfo *)p1)->ip_addr, \ + ((ConnectionInfo *)p2)->ip_addr); + if (res != 0) + { + return res; + } + + return ((ConnectionInfo *)p1)->port - \ + ((ConnectionInfo *)p2)->port; +} + +static void insert_into_sorted_servers(TrackerServerGroup *pTrackerGroup, \ + ConnectionInfo *pInsertedServer) +{ + ConnectionInfo *pDestServer; + for (pDestServer=pTrackerGroup->servers+pTrackerGroup->server_count; \ + pDestServer>pTrackerGroup->servers; pDestServer--) + { + if (storage_cmp_by_ip_and_port(pInsertedServer, \ + pDestServer-1) > 0) + { + memcpy(pDestServer, pInsertedServer, \ + sizeof(ConnectionInfo)); + return; + } + + memcpy(pDestServer, pDestServer-1, sizeof(ConnectionInfo)); + } + + memcpy(pDestServer, pInsertedServer, sizeof(ConnectionInfo)); +} + +static int copy_tracker_servers(TrackerServerGroup *pTrackerGroup, \ + const char *filename, char **ppTrackerServers) +{ + char **ppSrc; + char **ppEnd; + ConnectionInfo destServer; + char *pSeperator; + char szHost[128]; + int nHostLen; + + memset(&destServer, 0, sizeof(ConnectionInfo)); + destServer.sock = -1; + + ppEnd = ppTrackerServers + pTrackerGroup->server_count; + + pTrackerGroup->server_count = 0; + for (ppSrc=ppTrackerServers; ppSrc= sizeof(szHost)) + { + nHostLen = sizeof(szHost) - 1; + } + memcpy(szHost, *ppSrc, nHostLen); + szHost[nHostLen] = '\0'; + + if (getIpaddrByName(szHost, destServer.ip_addr, \ + sizeof(destServer.ip_addr)) == INADDR_NONE) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "host \"%s\" is invalid", \ + __LINE__, filename, szHost); + return EINVAL; + } + destServer.port = atoi(pSeperator+1); + if (destServer.port <= 0) + { + destServer.port = FDFS_TRACKER_SERVER_DEF_PORT; + } + + if (bsearch(&destServer, pTrackerGroup->servers, \ + pTrackerGroup->server_count, \ + sizeof(ConnectionInfo), \ + storage_cmp_by_ip_and_port) == NULL) + { + insert_into_sorted_servers(pTrackerGroup, &destServer); + pTrackerGroup->server_count++; + } + } + + /* + { + ConnectionInfo *pServer; + for (pServer=pTrackerGroup->servers; pServerservers+ \ + pTrackerGroup->server_count; pServer++) + { + //printf("server=%s:%d\n", \ + pServer->ip_addr, pServer->port); + } + } + */ + + return 0; +} + +int fdfs_load_tracker_group_ex(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename, IniContext *pIniContext) +{ + int result; + char *ppTrackerServers[FDFS_MAX_TRACKERS]; + + if ((pTrackerGroup->server_count=iniGetValues(NULL, "tracker_server", \ + pIniContext, ppTrackerServers, FDFS_MAX_TRACKERS)) <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "get item \"tracker_server\" fail", \ + __LINE__, conf_filename); + return ENOENT; + } + + pTrackerGroup->servers = (ConnectionInfo *)malloc( \ + sizeof(ConnectionInfo) * pTrackerGroup->server_count); + if (pTrackerGroup->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(ConnectionInfo) * \ + pTrackerGroup->server_count); + pTrackerGroup->server_count = 0; + return errno != 0 ? errno : ENOMEM; + } + + memset(pTrackerGroup->servers, 0, \ + sizeof(ConnectionInfo) * pTrackerGroup->server_count); + if ((result=copy_tracker_servers(pTrackerGroup, conf_filename, \ + ppTrackerServers)) != 0) + { + pTrackerGroup->server_count = 0; + free(pTrackerGroup->servers); + pTrackerGroup->servers = NULL; + return result; + } + + return 0; +} + +int fdfs_load_tracker_group(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename) +{ + IniContext iniContext; + int result; + + if ((result=iniLoadFromFile(conf_filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, ret code: %d", \ + __LINE__, conf_filename, result); + return result; + } + + result = fdfs_load_tracker_group_ex(pTrackerGroup, conf_filename, \ + &iniContext); + iniFreeContext(&iniContext); + + return result; +} + +static int fdfs_get_params_from_tracker(bool *use_storage_id) +{ + IniContext iniContext; + int result; + bool continue_flag; + + continue_flag = false; + if ((result=fdfs_get_ini_context_from_tracker(&g_tracker_group, \ + &iniContext, &continue_flag, false, NULL)) != 0) + { + return result; + } + + *use_storage_id = iniGetBoolValue(NULL, "use_storage_id", \ + &iniContext, false); + iniFreeContext(&iniContext); + + if (*use_storage_id) + { + result = fdfs_get_storage_ids_from_tracker_group( \ + &g_tracker_group); + } + + return result; +} + +static int fdfs_client_do_init_ex(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename, IniContext *iniContext) +{ + char *pBasePath; + int result; + bool use_storage_id; + bool load_fdfs_parameters_from_tracker; + + pBasePath = iniGetStrValue(NULL, "base_path", iniContext); + if (pBasePath == NULL) + { + strcpy(g_fdfs_base_path, "/tmp"); + } + else + { + snprintf(g_fdfs_base_path, sizeof(g_fdfs_base_path), + "%s", pBasePath); + chopPath(g_fdfs_base_path); + if (!fileExists(g_fdfs_base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" can't be accessed, error info: %s", \ + __LINE__, g_fdfs_base_path, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + if (!isDir(g_fdfs_base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" is not a directory!", \ + __LINE__, g_fdfs_base_path); + return ENOTDIR; + } + } + + g_fdfs_connect_timeout = iniGetIntValue(NULL, "connect_timeout", \ + iniContext, DEFAULT_CONNECT_TIMEOUT); + if (g_fdfs_connect_timeout <= 0) + { + g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT; + } + + g_fdfs_network_timeout = iniGetIntValue(NULL, "network_timeout", \ + iniContext, DEFAULT_NETWORK_TIMEOUT); + if (g_fdfs_network_timeout <= 0) + { + g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT; + } + + if ((result=fdfs_load_tracker_group_ex(pTrackerGroup, \ + conf_filename, iniContext)) != 0) + { + return result; + } + + g_anti_steal_token = iniGetBoolValue(NULL, \ + "http.anti_steal.check_token", \ + iniContext, false); + if (g_anti_steal_token) + { + char *anti_steal_secret_key; + + anti_steal_secret_key = iniGetStrValue(NULL, \ + "http.anti_steal.secret_key", \ + iniContext); + if (anti_steal_secret_key == NULL || \ + *anti_steal_secret_key == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "param \"http.anti_steal.secret_key\""\ + " not exist or is empty", __LINE__); + return EINVAL; + } + + buffer_strcpy(&g_anti_steal_secret_key, anti_steal_secret_key); + } + + g_tracker_server_http_port = iniGetIntValue(NULL, \ + "http.tracker_server_port", \ + iniContext, 80); + if (g_tracker_server_http_port <= 0) + { + g_tracker_server_http_port = 80; + } + + if ((result=fdfs_connection_pool_init(conf_filename, iniContext)) != 0) + { + return result; + } + + load_fdfs_parameters_from_tracker = iniGetBoolValue(NULL, \ + "load_fdfs_parameters_from_tracker", \ + iniContext, false); + if (load_fdfs_parameters_from_tracker) + { + fdfs_get_params_from_tracker(&use_storage_id); + } + else + { + use_storage_id = iniGetBoolValue(NULL, "use_storage_id", \ + iniContext, false); + if (use_storage_id) + { + result = fdfs_load_storage_ids_from_file( \ + conf_filename, iniContext); + } + } + +#ifdef DEBUG_FLAG + logDebug("base_path=%s, " \ + "connect_timeout=%d, "\ + "network_timeout=%d, "\ + "tracker_server_count=%d, " \ + "anti_steal_token=%d, " \ + "anti_steal_secret_key length=%d, " \ + "use_connection_pool=%d, " \ + "g_connection_pool_max_idle_time=%ds, " \ + "use_storage_id=%d, storage server id count: %d\n", \ + g_fdfs_base_path, g_fdfs_connect_timeout, \ + g_fdfs_network_timeout, pTrackerGroup->server_count, \ + g_anti_steal_token, g_anti_steal_secret_key.length, \ + g_use_connection_pool, g_connection_pool_max_idle_time, \ + use_storage_id, g_storage_id_count); +#endif + + return 0; +} + +int fdfs_client_init_from_buffer_ex(TrackerServerGroup *pTrackerGroup, \ + const char *buffer) +{ + IniContext iniContext; + char *new_buff; + int result; + + new_buff = strdup(buffer); + if (new_buff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "strdup %d bytes fail", __LINE__, (int)strlen(buffer)); + return ENOMEM; + } + + result = iniLoadFromBuffer(new_buff, &iniContext); + free(new_buff); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load parameters from buffer fail, ret code: %d", \ + __LINE__, result); + return result; + } + + result = fdfs_client_do_init_ex(pTrackerGroup, "buffer", &iniContext); + iniFreeContext(&iniContext); + return result; +} + +int fdfs_client_init_ex(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename) +{ + IniContext iniContext; + int result; + + if ((result=iniLoadFromFile(conf_filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, ret code: %d", \ + __LINE__, conf_filename, result); + return result; + } + + result = fdfs_client_do_init_ex(pTrackerGroup, conf_filename, \ + &iniContext); + iniFreeContext(&iniContext); + return result; +} + +int fdfs_copy_tracker_group(TrackerServerGroup *pDestTrackerGroup, \ + TrackerServerGroup *pSrcTrackerGroup) +{ + int bytes; + ConnectionInfo *pDestServer; + ConnectionInfo *pDestServerEnd; + + bytes = sizeof(ConnectionInfo) * pSrcTrackerGroup->server_count; + pDestTrackerGroup->servers = (ConnectionInfo *)malloc(bytes); + if (pDestTrackerGroup->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, bytes); + return errno != 0 ? errno : ENOMEM; + } + + pDestTrackerGroup->server_index = 0; + pDestTrackerGroup->server_count = pSrcTrackerGroup->server_count; + memcpy(pDestTrackerGroup->servers, pSrcTrackerGroup->servers, bytes); + + pDestServerEnd = pDestTrackerGroup->servers + \ + pDestTrackerGroup->server_count; + for (pDestServer=pDestTrackerGroup->servers; \ + pDestServersock = -1; + } + + return 0; +} + +bool fdfs_tracker_group_equals(TrackerServerGroup *pGroup1, \ + TrackerServerGroup *pGroup2) +{ + ConnectionInfo *pServer1; + ConnectionInfo *pServer2; + ConnectionInfo *pEnd1; + + if (pGroup1->server_count != pGroup1->server_count) + { + return false; + } + + pEnd1 = pGroup1->servers + pGroup1->server_count; + pServer1 = pGroup1->servers; + pServer2 = pGroup2->servers; + while (pServer1 < pEnd1) + { + if (!(strcmp(pServer1->ip_addr, pServer2->ip_addr) == 0 && + pServer1->port == pServer2->port)) + { + return false; + } + + pServer1++; + pServer2++; + } + + return true; +} + +void fdfs_client_destroy_ex(TrackerServerGroup *pTrackerGroup) +{ + if (pTrackerGroup->servers != NULL) + { + free(pTrackerGroup->servers); + pTrackerGroup->servers = NULL; + + pTrackerGroup->server_count = 0; + pTrackerGroup->server_index = 0; + } +} + +const char *fdfs_get_file_ext_name_ex(const char *filename, + const bool twoExtName) +{ + const char *fileExtName; + const char *p; + const char *pStart; + int extNameLen; + + fileExtName = strrchr(filename, '.'); + if (fileExtName == NULL) + { + return NULL; + } + + extNameLen = strlen(fileExtName + 1); + if (extNameLen > FDFS_FILE_EXT_NAME_MAX_LEN) + { + return NULL; + } + + if (strchr(fileExtName + 1, '/') != NULL) //invalid extension name + { + return NULL; + } + + if (!twoExtName) + { + return fileExtName + 1; + } + + pStart = fileExtName - (FDFS_FILE_EXT_NAME_MAX_LEN - extNameLen) - 1; + if (pStart < filename) + { + pStart = filename; + } + + p = fileExtName - 1; //before . + while ((p > pStart) && (*p != '.')) + { + p--; + } + + if (p > pStart) //found (extension name have a dot) + { + if (strchr(p + 1, '/') == NULL) //valid extension name + { + return p + 1; //skip . + } + } + + return fileExtName + 1; //skip . +} + diff --git a/client/client_func.h b/client/client_func.h new file mode 100644 index 0000000..fe83ca7 --- /dev/null +++ b/client/client_func.h @@ -0,0 +1,145 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//client_func.h + +#include "tracker_types.h" +#include "client_global.h" +#include "ini_file_reader.h" + +#ifndef _CLIENT_FUNC_H_ +#define _CLIENT_FUNC_H_ + +typedef struct { + time_t create_timestamp; + int crc32; + int source_id; //source storage id + int64_t file_size; + char source_ip_addr[IP_ADDRESS_SIZE]; //source storage ip address +} FDFSFileInfo; + +#ifdef __cplusplus +extern "C" { +#endif + +#define fdfs_client_init(filename) \ + fdfs_client_init_ex((&g_tracker_group), filename) + +#define fdfs_client_init_from_buffer(buffer) \ + fdfs_client_init_from_buffer_ex((&g_tracker_group), buffer) + +#define fdfs_client_destroy() \ + fdfs_client_destroy_ex((&g_tracker_group)) + +/** +* client initial from config file +* params: +* pTrackerGroup: tracker group +* conf_filename: client config filename +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_client_init_ex(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename); + + +/** +* client initial from buffer +* params: +* pTrackerGroup: tracker group +* conf_filename: client config filename +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_client_init_from_buffer_ex(TrackerServerGroup *pTrackerGroup, \ + const char *buffer); + +/** +* load tracker server group +* params: +* pTrackerGroup: tracker group +* conf_filename: tracker server group config filename +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_load_tracker_group(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename); + +/** +* load tracker server group +* params: +* pTrackerGroup: tracker group +* conf_filename: config filename +* items: ini file items +* nItemCount: ini file item count +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_load_tracker_group_ex(TrackerServerGroup *pTrackerGroup, \ + const char *conf_filename, IniContext *pIniContext); + +/** +* copy tracker server group +* params: +* pDestTrackerGroup: the dest tracker group +* pSrcTrackerGroup: the source tracker group +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_copy_tracker_group(TrackerServerGroup *pDestTrackerGroup, \ + TrackerServerGroup *pSrcTrackerGroup); + +/** +* client destroy function +* params: +* pTrackerGroup: tracker group +* return: none +**/ +void fdfs_client_destroy_ex(TrackerServerGroup *pTrackerGroup); + +/** +* tracker group equals +* params: +* pGroup1: tracker group 1 +* pGroup2: tracker group 2 +* return: true for equals, otherwise false +**/ +bool fdfs_tracker_group_equals(TrackerServerGroup *pGroup1, \ + TrackerServerGroup *pGroup2); + +/** +* get file ext name from filename, extension name do not include dot +* params: +* filename: the filename +* return: file ext name, NULL for no ext name +**/ +#define fdfs_get_file_ext_name1(filename) \ + fdfs_get_file_ext_name_ex(filename, false) + +/** +* get file ext name from filename, extension name maybe include dot +* params: +* filename: the filename +* return: file ext name, NULL for no ext name +**/ +#define fdfs_get_file_ext_name2(filename) \ + fdfs_get_file_ext_name_ex(filename, true) + +#define fdfs_get_file_ext_name(filename) \ + fdfs_get_file_ext_name_ex(filename, true) + +/** +* get file ext name from filename +* params: +* filename: the filename +* twoExtName: two extension name as the extension name +* return: file ext name, NULL for no ext name +**/ +const char *fdfs_get_file_ext_name_ex(const char *filename, + const bool twoExtName); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/client/client_global.c b/client/client_global.c new file mode 100644 index 0000000..19964b4 --- /dev/null +++ b/client/client_global.c @@ -0,0 +1,18 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include "client_global.h" + +int g_tracker_server_http_port = 80; +TrackerServerGroup g_tracker_group = {0, 0, -1, NULL}; + +bool g_anti_steal_token = false; +BufferInfo g_anti_steal_secret_key = {0}; + diff --git a/client/client_global.h b/client/client_global.h new file mode 100644 index 0000000..e0e1743 --- /dev/null +++ b/client/client_global.h @@ -0,0 +1,35 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//client_global.h + +#ifndef _CLIENT_GLOBAL_H +#define _CLIENT_GLOBAL_H + +#include "common_define.h" +#include "tracker_types.h" +#include "fdfs_shared_func.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_tracker_server_http_port; +extern TrackerServerGroup g_tracker_group; + +extern bool g_anti_steal_token; +extern BufferInfo g_anti_steal_secret_key; + +#define fdfs_get_tracker_leader_index(leaderIp, leaderPort) \ + fdfs_get_tracker_leader_index_ex(&g_tracker_group, \ + leaderIp, leaderPort) +#ifdef __cplusplus +} +#endif + +#endif diff --git a/client/fdfs_append_file.c b/client/fdfs_append_file.c new file mode 100644 index 0000000..fd539fa --- /dev/null +++ b/client/fdfs_append_file.c @@ -0,0 +1,66 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + ConnectionInfo *pTrackerServer; + int result; + char appender_file_id[128]; + + if (argc < 4) + { + printf("Usage: %s " \ + "\n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_ERR; + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + snprintf(appender_file_id, sizeof(appender_file_id), "%s", argv[2]); + local_filename = argv[3]; + if ((result=storage_append_by_filename1(pTrackerServer, \ + NULL, local_filename, appender_file_id)) != 0) + { + printf("append file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + + return result; +} + diff --git a/client/fdfs_appender_test.c b/client/fdfs_appender_test.c new file mode 100644 index 0000000..f9d2876 --- /dev/null +++ b/client/fdfs_appender_test.c @@ -0,0 +1,439 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "fdfs_global.h" +#include "base64.h" +#include "sockopt.h" +#include "logger.h" +#include "fdfs_http_shared.h" + +int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \ + const int current_size) +{ + if (arg == NULL) + { + return EINVAL; + } + + if (fwrite(data, current_size, 1, (FILE *)arg) != 1) + { + return errno != 0 ? errno : EIO; + } + + return 0; +} + +int uploadFileCallback(void *arg, const int64_t file_size, int sock) +{ + int64_t total_send_bytes; + char *filename; + + if (arg == NULL) + { + return EINVAL; + } + + filename = (char *)arg; + return tcpsendfile(sock, filename, file_size, \ + g_fdfs_network_timeout, &total_send_bytes); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + ConnectionInfo storageServer; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[256]; + char appender_filename[256]; + FDFSMetaData meta_list[32]; + int meta_count; + char token[32 + 1]; + char file_id[128]; + char file_url[256]; + char szDatetime[20]; + char szPortPart[16]; + int url_len; + time_t ts; + int64_t file_offset; + int64_t file_size = 0; + int store_path_index; + FDFSFileInfo file_info; + int upload_type; + const char *file_ext_name; + struct stat stat_buf; + + printf("This is FastDFS client test program v%d.%02d\n" \ +"\nCopyright (C) 2008, Happy Fish / YuQing\n" \ +"\nFastDFS may be copied only under the terms of the GNU General\n" \ +"Public License V3, which may be found in the FastDFS source kit.\n" \ +"Please visit the FastDFS Home Page http://www.csource.org/ \n" \ +"for more detail.\n\n" \ +, g_fdfs_version.major, g_fdfs_version.minor); + + if (argc < 3) + { + printf("Usage: %s " \ + "[FILE | BUFF | CALLBACK]\n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_DEBUG; + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + local_filename = argv[2]; + if (argc == 3) + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + else + { + if (strcmp(argv[3], "BUFF") == 0) + { + upload_type = FDFS_UPLOAD_BY_BUFF; + } + else if (strcmp(argv[3], "CALLBACK") == 0) + { + upload_type = FDFS_UPLOAD_BY_CALLBACK; + } + else + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + } + + *group_name = '\0'; + store_path_index = 0; + if ((result=tracker_query_storage_store(pTrackerServer, \ + &storageServer, group_name, &store_path_index)) != 0) + { + fdfs_client_destroy(); + printf("tracker_query_storage fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + printf("group_name=%s, ip_addr=%s, port=%d\n", \ + group_name, storageServer.ip_addr, \ + storageServer.port); + + if ((pStorageServer=tracker_connect_server(&storageServer, \ + &result)) == NULL) + { + fdfs_client_destroy(); + return result; + } + + memset(&meta_list, 0, sizeof(meta_list)); + meta_count = 0; + strcpy(meta_list[meta_count].name, "ext_name"); + strcpy(meta_list[meta_count].value, "jpg"); + meta_count++; + strcpy(meta_list[meta_count].name, "width"); + strcpy(meta_list[meta_count].value, "160"); + meta_count++; + strcpy(meta_list[meta_count].name, "height"); + strcpy(meta_list[meta_count].value, "80"); + meta_count++; + strcpy(meta_list[meta_count].name, "file_size"); + strcpy(meta_list[meta_count].value, "115120"); + meta_count++; + + file_ext_name = fdfs_get_file_ext_name(local_filename); + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_appender_by_filename ( \ + pTrackerServer, pStorageServer, \ + store_path_index, local_filename, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_upload_appender_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_upload_appender_by_filebuff( \ + pTrackerServer, pStorageServer, \ + store_path_index, file_content, \ + file_size, file_ext_name, \ + meta_list, meta_count, \ + group_name, remote_filename); + free(file_content); + } + + printf("storage_upload_appender_by_filebuff\n"); + } + else + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_appender_by_callback( \ + pTrackerServer, pStorageServer, \ + store_path_index, uploadFileCallback, \ + local_filename, file_size, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_upload_appender_by_callback\n"); + } + + if (result != 0) + { + printf("upload file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + if (g_tracker_server_http_port == 80) + { + *szPortPart = '\0'; + } + else + { + sprintf(szPortPart, ":%d", g_tracker_server_http_port); + } + + sprintf(file_id, "%s/%s", group_name, remote_filename); + url_len = sprintf(file_url, "http://%s%s/%s", \ + pTrackerServer->ip_addr, szPortPart, file_id); + if (g_anti_steal_token) + { + ts = time(NULL); + fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \ + ts, token); + sprintf(file_url + url_len, "?token=%s&ts=%d", token, (int)ts); + } + + printf("group_name=%s, remote_filename=%s\n", \ + group_name, remote_filename); + + fdfs_get_file_info(group_name, remote_filename, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("file url: %s\n", file_url); + + //sleep(90); + strcpy(appender_filename, remote_filename); + if (storage_truncate_file(pTrackerServer, pStorageServer, \ + group_name, appender_filename, file_size / 2) != 0) + { + printf("truncate file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + fdfs_get_file_info(group_name, appender_filename, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("file url: %s\n", file_url); + if (file_info.file_size != file_size / 2) + { + fprintf(stderr, "file size: "INT64_PRINTF_FORMAT \ + " != "INT64_PRINTF_FORMAT"!!!", file_info.file_size, file_size / 2); + } + + //sleep(100); + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_append_by_filename(pTrackerServer, \ + pStorageServer, local_filename, + group_name, appender_filename); + + printf("storage_append_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_append_by_filebuff(pTrackerServer, \ + pStorageServer, file_content, \ + file_size, group_name, appender_filename); + free(file_content); + } + + printf("storage_append_by_filebuff\n"); + } + else + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_append_by_callback(pTrackerServer, \ + pStorageServer, uploadFileCallback, \ + local_filename, file_size, \ + group_name, appender_filename); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_append_by_callback\n"); + } + + if (result != 0) + { + printf("append file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + printf("append file successfully.\n"); + fdfs_get_file_info(group_name, remote_filename, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + if (file_info.file_size != file_size + file_size / 2) + { + fprintf(stderr, "file size: "INT64_PRINTF_FORMAT \ + " != "INT64_PRINTF_FORMAT"!!!", file_info.file_size, \ + file_size + file_size / 2); + } + + file_offset = file_info.file_size; + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_modify_by_filename(pTrackerServer, \ + pStorageServer, local_filename, \ + file_offset, group_name, \ + appender_filename); + + printf("storage_modify_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_modify_by_filebuff(pTrackerServer, \ + pStorageServer, file_content, \ + file_offset, file_size, group_name, \ + appender_filename); + free(file_content); + } + + printf("storage_modify_by_filebuff\n"); + } + else + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_modify_by_callback(pTrackerServer, \ + pStorageServer, uploadFileCallback, \ + local_filename, file_offset, \ + file_size, group_name, appender_filename); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_modify_by_callback\n"); + } + + if (result != 0) + { + printf("modify file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + printf("modify file successfully.\n"); + fdfs_get_file_info(group_name, remote_filename, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + if (file_info.file_size != 2 * file_size + file_size / 2) + { + fprintf(stderr, "file size: "INT64_PRINTF_FORMAT \ + " != "INT64_PRINTF_FORMAT"!!!", file_info.file_size, \ + 2 * file_size + file_size /2); + } + + tracker_disconnect_server_ex(pStorageServer, true); + tracker_disconnect_server_ex(pTrackerServer, true); + + fdfs_client_destroy(); + + return result; +} + diff --git a/client/fdfs_appender_test1.c b/client/fdfs_appender_test1.c new file mode 100644 index 0000000..e535e38 --- /dev/null +++ b/client/fdfs_appender_test1.c @@ -0,0 +1,435 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "fdfs_global.h" +#include "base64.h" +#include "sockopt.h" +#include "logger.h" +#include "fdfs_http_shared.h" + +int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \ + const int current_size) +{ + if (arg == NULL) + { + return EINVAL; + } + + if (fwrite(data, current_size, 1, (FILE *)arg) != 1) + { + return errno != 0 ? errno : EIO; + } + + return 0; +} + +int uploadFileCallback(void *arg, const int64_t file_size, int sock) +{ + int64_t total_send_bytes; + char *filename; + + if (arg == NULL) + { + return EINVAL; + } + + filename = (char *)arg; + return tcpsendfile(sock, filename, file_size, \ + g_fdfs_network_timeout, &total_send_bytes); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + ConnectionInfo storageServer; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char file_id[256]; + char appender_file_id[256]; + FDFSMetaData meta_list[32]; + int meta_count; + char token[32 + 1]; + char file_url[256]; + char szDatetime[20]; + char szPortPart[16]; + int url_len; + time_t ts; + int64_t file_offset; + int64_t file_size = 0; + int store_path_index; + FDFSFileInfo file_info; + int upload_type; + const char *file_ext_name; + struct stat stat_buf; + + printf("This is FastDFS client test program v%d.%02d\n" \ +"\nCopyright (C) 2008, Happy Fish / YuQing\n" \ +"\nFastDFS may be copied only under the terms of the GNU General\n" \ +"Public License V3, which may be found in the FastDFS source kit.\n" \ +"Please visit the FastDFS Home Page http://www.csource.org/ \n" \ +"for more detail.\n\n" \ +, g_fdfs_version.major, g_fdfs_version.minor); + + if (argc < 3) + { + printf("Usage: %s " \ + "[FILE | BUFF | CALLBACK]\n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_DEBUG; + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + local_filename = argv[2]; + if (argc == 3) + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + else + { + if (strcmp(argv[3], "BUFF") == 0) + { + upload_type = FDFS_UPLOAD_BY_BUFF; + } + else if (strcmp(argv[3], "CALLBACK") == 0) + { + upload_type = FDFS_UPLOAD_BY_CALLBACK; + } + else + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + } + + store_path_index = 0; + *group_name = '\0'; + if ((result=tracker_query_storage_store(pTrackerServer, \ + &storageServer, group_name, &store_path_index)) != 0) + { + fdfs_client_destroy(); + printf("tracker_query_storage fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + printf("group_name=%s, ip_addr=%s, port=%d\n", \ + group_name, storageServer.ip_addr, \ + storageServer.port); + + if ((pStorageServer=tracker_connect_server(&storageServer, \ + &result)) == NULL) + { + fdfs_client_destroy(); + return result; + } + + memset(&meta_list, 0, sizeof(meta_list)); + meta_count = 0; + strcpy(meta_list[meta_count].name, "ext_name"); + strcpy(meta_list[meta_count].value, "jpg"); + meta_count++; + strcpy(meta_list[meta_count].name, "width"); + strcpy(meta_list[meta_count].value, "160"); + meta_count++; + strcpy(meta_list[meta_count].name, "height"); + strcpy(meta_list[meta_count].value, "80"); + meta_count++; + strcpy(meta_list[meta_count].name, "file_size"); + strcpy(meta_list[meta_count].value, "115120"); + meta_count++; + + file_ext_name = fdfs_get_file_ext_name(local_filename); + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_appender_by_filename1( \ + pTrackerServer, pStorageServer, \ + store_path_index, local_filename, \ + file_ext_name, meta_list, meta_count, \ + group_name, file_id); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_upload_appender_by_filename1\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_upload_appender_by_filebuff1( \ + pTrackerServer, pStorageServer, \ + store_path_index, file_content, \ + file_size, file_ext_name, \ + meta_list, meta_count, \ + group_name, file_id); + free(file_content); + } + + printf("storage_upload_appender_by_filebuff1\n"); + } + else + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_appender_by_callback1( \ + pTrackerServer, pStorageServer, \ + store_path_index, uploadFileCallback, \ + local_filename, file_size, \ + file_ext_name, meta_list, meta_count, \ + group_name, file_id); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_upload_appender_by_callback1\n"); + } + + if (result != 0) + { + printf("upload file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + if (g_tracker_server_http_port == 80) + { + *szPortPart = '\0'; + } + else + { + sprintf(szPortPart, ":%d", g_tracker_server_http_port); + } + + url_len = sprintf(file_url, "http://%s%s/%s", \ + pTrackerServer->ip_addr, szPortPart, file_id); + if (g_anti_steal_token) + { + ts = time(NULL); + fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \ + ts, token); + sprintf(file_url + url_len, "?token=%s&ts=%d", token, (int)ts); + } + + printf("fild_id=%s\n", file_id); + + fdfs_get_file_info1(file_id, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("file url: %s\n", file_url); + + + strcpy(appender_file_id, file_id); + if (storage_truncate_file1(pTrackerServer, pStorageServer, \ + appender_file_id, 0) != 0) + { + printf("truncate file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + fdfs_get_file_info1(file_id, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("file url: %s\n", file_url); + if (file_info.file_size != 0) + { + fprintf(stderr, "file size: "INT64_PRINTF_FORMAT \ + " != 0!!!", file_info.file_size); + } + + //sleep(70); + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_append_by_filename1(pTrackerServer, \ + pStorageServer, local_filename, + appender_file_id); + + printf("storage_append_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_append_by_filebuff1(pTrackerServer, \ + pStorageServer, file_content, \ + file_size, appender_file_id); + free(file_content); + } + + printf("storage_append_by_filebuff1\n"); + } + else + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_append_by_callback1(pTrackerServer, \ + pStorageServer, uploadFileCallback, \ + local_filename, file_size, \ + appender_file_id); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_append_by_callback1\n"); + } + + if (result != 0) + { + printf("append file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + printf("append file successfully.\n"); + fdfs_get_file_info1(appender_file_id, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + if (file_info.file_size != file_size) + { + fprintf(stderr, "file size: "INT64_PRINTF_FORMAT \ + " != "INT64_PRINTF_FORMAT"!!!", file_info.file_size, \ + file_size); + } + + file_offset = file_size; + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_modify_by_filename1(pTrackerServer, \ + pStorageServer, local_filename, + file_offset, appender_file_id); + + printf("storage_modify_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_modify_by_filebuff1( \ + pTrackerServer, pStorageServer, \ + file_content, file_offset, file_size, \ + appender_file_id); + free(file_content); + } + + printf("storage_modify_by_filebuff1\n"); + } + else + { + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_modify_by_callback1( \ + pTrackerServer, pStorageServer, \ + uploadFileCallback, \ + local_filename, file_offset, \ + file_size, appender_file_id); + } + else + { + result = errno != 0 ? errno : ENOENT; + } + + printf("storage_modify_by_callback1\n"); + } + + if (result != 0) + { + printf("modify file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + printf("modify file successfully.\n"); + fdfs_get_file_info1(appender_file_id, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + if (file_info.file_size != 2 * file_size) + { + fprintf(stderr, "file size: "INT64_PRINTF_FORMAT \ + " != "INT64_PRINTF_FORMAT"!!!", file_info.file_size, \ + 2 * file_size); + } + + tracker_disconnect_server_ex(pStorageServer, true); + tracker_disconnect_server_ex(pTrackerServer, true); + + fdfs_client_destroy(); + + return result; +} + diff --git a/client/fdfs_client.h b/client/fdfs_client.h new file mode 100644 index 0000000..d63e792 --- /dev/null +++ b/client/fdfs_client.h @@ -0,0 +1,22 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef FDFS_CLIENT_H +#define FDFS_CLIENT_H + +#include "shared_func.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_client.h" +#include "storage_client.h" +#include "storage_client1.h" +#include "client_func.h" +#include "client_global.h" + +#endif + diff --git a/client/fdfs_crc32.c b/client/fdfs_crc32.c new file mode 100644 index 0000000..b460197 --- /dev/null +++ b/client/fdfs_crc32.c @@ -0,0 +1,105 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hash.h" + +int main(int argc, char *argv[]) +{ + int64_t file_size; + int64_t remain_bytes; + char *filename; + int fd; + int read_bytes; + int result; + int crc32; + char buff[512 * 1024]; + + if (argc < 2) + { + printf("Usage: %s \n", argv[0]); + return 1; + } + + filename = argv[1]; + fd = open(filename, O_RDONLY); + if (fd < 0) + { + printf("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s\n", \ + __LINE__, filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + if ((file_size=lseek(fd, 0, SEEK_END)) < 0) + { + printf("file: "__FILE__", line: %d, " \ + "call lseek fail, " \ + "errno: %d, error info: %s\n", \ + __LINE__, errno, STRERROR(errno)); + close(fd); + return errno; + } + + if (lseek(fd, 0, SEEK_SET) < 0) + { + printf("file: "__FILE__", line: %d, " \ + "call lseek fail, " \ + "errno: %d, error info: %s\n", \ + __LINE__, errno, STRERROR(errno)); + close(fd); + return errno; + } + + crc32 = CRC32_XINIT; + result = 0; + remain_bytes = file_size; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + read_bytes = sizeof(buff); + } + else + { + read_bytes = remain_bytes; + } + + if (read(fd, buff, read_bytes) != read_bytes) + { + printf("file: "__FILE__", line: %d, " \ + "call lseek fail, " \ + "errno: %d, error info: %s\n", \ + __LINE__, errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + break; + } + + crc32 = CRC32_ex(buff, read_bytes, crc32); + remain_bytes -= read_bytes; + } + + close(fd); + + if (result == 0) + { + crc32 = CRC32_FINAL(crc32); + printf("%u\n", crc32); + } + + return result; +} + diff --git a/client/fdfs_delete_file.c b/client/fdfs_delete_file.c new file mode 100644 index 0000000..da3ce1a --- /dev/null +++ b/client/fdfs_delete_file.c @@ -0,0 +1,62 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" + +int main(int argc, char *argv[]) +{ + char *conf_filename; + ConnectionInfo *pTrackerServer; + int result; + char file_id[128]; + + if (argc < 3) + { + printf("Usage: %s \n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_ERR; + ignore_signal_pipe(); + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + snprintf(file_id, sizeof(file_id), "%s", argv[2]); + if ((result=storage_delete_file1(pTrackerServer, NULL, file_id)) != 0) + { + printf("delete file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + + return result; +} + diff --git a/client/fdfs_download_file.c b/client/fdfs_download_file.c new file mode 100644 index 0000000..23c2775 --- /dev/null +++ b/client/fdfs_download_file.c @@ -0,0 +1,97 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + ConnectionInfo *pTrackerServer; + int result; + char file_id[128]; + int64_t file_size; + int64_t file_offset; + int64_t download_bytes; + + if (argc < 3) + { + printf("Usage: %s " \ + "[local_filename] [ " \ + "]\n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_ERR; + ignore_signal_pipe(); + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + snprintf(file_id, sizeof(file_id), "%s", argv[2]); + + file_offset = 0; + download_bytes = 0; + if (argc >= 4) + { + local_filename = argv[3]; + if (argc >= 6) + { + file_offset = strtoll(argv[4], NULL, 10); + download_bytes = strtoll(argv[5], NULL, 10); + } + } + else + { + local_filename = strrchr(file_id, '/'); + if (local_filename != NULL) + { + local_filename++; //skip / + } + else + { + local_filename = file_id; + } + } + + result = storage_do_download_file1_ex(pTrackerServer, \ + NULL, FDFS_DOWNLOAD_TO_FILE, file_id, \ + file_offset, download_bytes, \ + &local_filename, NULL, &file_size); + if (result != 0) + { + printf("download file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + + return 0; +} + diff --git a/client/fdfs_file_info.c b/client/fdfs_file_info.c new file mode 100644 index 0000000..6f3a2d4 --- /dev/null +++ b/client/fdfs_file_info.c @@ -0,0 +1,71 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char file_id[128]; + int result; + FDFSFileInfo file_info; + + if (argc < 3) + { + printf("Usage: %s \n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_ERR; + ignore_signal_pipe(); + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + snprintf(file_id, sizeof(file_id), "%s", argv[2]); + memset(&file_info, 0, sizeof(file_info)); + result = fdfs_get_file_info_ex1(file_id, true, &file_info); + if (result != 0) + { + printf("query file info fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + else + { + char szDatetime[32]; + + printf("source storage id: %d\n", file_info.source_id); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file create timestamp: %s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size: "INT64_PRINTF_FORMAT"\n", \ + file_info.file_size); + printf("file crc32: %u (0x%08X)\n", \ + file_info.crc32, file_info.crc32); + } + + tracker_close_all_connections(); + fdfs_client_destroy(); + + return 0; +} + diff --git a/client/fdfs_link_library.sh b/client/fdfs_link_library.sh new file mode 100755 index 0000000..bab18e3 --- /dev/null +++ b/client/fdfs_link_library.sh @@ -0,0 +1,28 @@ +tmp_src_filename=_fdfs_check_bits_.c +cat < $tmp_src_filename +#include +#include +#include +int main() +{ + printf("%d\n", (int)sizeof(long)); + return 0; +} +EOF + +gcc -D_FILE_OFFSET_BITS=64 -o a.out $tmp_src_filename +OS_BITS=`./a.out` + +rm $tmp_src_filename a.out + +TARGET_LIB="/usr/local/lib" +if [ "`id -u`" = "0" ]; then + ln -fs $TARGET_LIB/libfastcommon.so.1 /usr/lib/libfastcommon.so + ln -fs $TARGET_LIB/libfdfsclient.so.1 /usr/lib/libfdfsclient.so + + if [ "$OS_BITS" = "8" ]; then + ln -fs $TARGET_LIB/libfastcommon.so.1 /usr/lib64/libfastcommon.so + ln -fs $TARGET_LIB/libfdfsclient.so.1 /usr/lib64/libfdfsclient.so + fi +fi + diff --git a/client/fdfs_link_library.sh.in b/client/fdfs_link_library.sh.in new file mode 100755 index 0000000..b04f762 --- /dev/null +++ b/client/fdfs_link_library.sh.in @@ -0,0 +1,28 @@ +tmp_src_filename=_fdfs_check_bits_.c +cat < $tmp_src_filename +#include +#include +#include +int main() +{ + printf("%d\n", (int)sizeof(long)); + return 0; +} +EOF + +gcc -D_FILE_OFFSET_BITS=64 -o a.out $tmp_src_filename +OS_BITS=`./a.out` + +rm $tmp_src_filename a.out + +TARGET_LIB="$(TARGET_PREFIX)/lib" +if [ "`id -u`" = "0" ]; then + ln -fs $TARGET_LIB/libfastcommon.so.1 /usr/lib/libfastcommon.so + ln -fs $TARGET_LIB/libfdfsclient.so.1 /usr/lib/libfdfsclient.so + + if [ "$OS_BITS" = "8" ]; then + ln -fs $TARGET_LIB/libfastcommon.so.1 /usr/lib64/libfastcommon.so + ln -fs $TARGET_LIB/libfdfsclient.so.1 /usr/lib64/libfdfsclient.so + fi +fi + diff --git a/client/fdfs_monitor.c b/client/fdfs_monitor.c new file mode 100644 index 0000000..708d362 --- /dev/null +++ b/client/fdfs_monitor.c @@ -0,0 +1,571 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "sockopt.h" +#include "logger.h" +#include "client_global.h" +#include "fdfs_global.h" +#include "fdfs_client.h" + +static ConnectionInfo *pTrackerServer; + +static int list_all_groups(const char *group_name); + +static void usage(char *argv[]) +{ + printf("Usage: %s [-h ] [list|delete|set_trunk_server " \ + "[storage_id]]\n", argv[0]); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + int result; + char *op_type; + char *tracker_server; + int arg_index; + char *group_name; + + if (argc < 2) + { + usage(argv); + return 1; + } + + tracker_server = NULL; + conf_filename = argv[1]; + arg_index = 2; + + if (arg_index >= argc) + { + op_type = "list"; + } + else + { + int len; + + len = strlen(argv[arg_index]); + if (len >= 2 && strncmp(argv[arg_index], "-h", 2) == 0) + { + if (len == 2) + { + arg_index++; + if (arg_index >= argc) + { + usage(argv); + return 1; + } + + tracker_server = argv[arg_index++]; + } + else + { + tracker_server = argv[arg_index] + 2; + arg_index++; + } + + if (arg_index < argc) + { + op_type = argv[arg_index++]; + } + else + { + op_type = "list"; + } + } + else + { + op_type = argv[arg_index++]; + } + } + + log_init(); + g_log_context.log_level = LOG_DEBUG; + ignore_signal_pipe(); + + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + load_log_level_ex(conf_filename); + + if (tracker_server == NULL) + { + if (g_tracker_group.server_count > 1) + { + srand(time(NULL)); + rand(); //discard the first + g_tracker_group.server_index = (int)( \ + (g_tracker_group.server_count * (double)rand()) \ + / (double)RAND_MAX); + } + } + else + { + int i; + char ip_addr[IP_ADDRESS_SIZE]; + + *ip_addr = '\0'; + if (getIpaddrByName(tracker_server, ip_addr, sizeof(ip_addr)) \ + == INADDR_NONE) + { + printf("resolve ip address of tracker server: %s " \ + "fail!\n", tracker_server); + return 2; + } + + for (i=0; iip_addr, pTrackerServer->port); + + if (arg_index < argc) + { + group_name = argv[arg_index++]; + } + else + { + group_name = NULL; + } + + if (strcmp(op_type, "list") == 0) + { + if (group_name == NULL) + { + result = list_all_groups(NULL); + } + else + { + result = list_all_groups(group_name); + } + } + else if (strcmp(op_type, "delete") == 0) + { + char *storage_id; + if (arg_index >= argc) + { + usage(argv); + return 1; + } + + storage_id = argv[arg_index++]; + + if ((result=tracker_delete_storage(&g_tracker_group, \ + group_name, storage_id)) == 0) + { + printf("delete storage server %s::%s success\n", \ + group_name, storage_id); + } + else + { + printf("delete storage server %s::%s fail, " \ + "error no: %d, error info: %s\n", \ + group_name, storage_id, \ + result, STRERROR(result)); + } + } + else if (strcmp(op_type, "set_trunk_server") == 0) + { + char *storage_id; + char new_trunk_server_id[FDFS_STORAGE_ID_MAX_SIZE]; + + if (group_name == NULL) + { + usage(argv); + return 1; + } + if (arg_index >= argc) + { + storage_id = ""; + } + else + { + storage_id = argv[arg_index++]; + } + + if ((result=tracker_set_trunk_server(&g_tracker_group, \ + group_name, storage_id, new_trunk_server_id)) == 0) + { + printf("set trunk server %s::%s success, " \ + "new trunk server: %s\n", group_name, \ + storage_id, new_trunk_server_id); + } + else + { + printf("set trunk server %s::%s fail, " \ + "error no: %d, error info: %s\n", \ + group_name, storage_id, \ + result, STRERROR(result)); + } + } + else + { + printf("Invalid command %s\n\n", op_type); + usage(argv); + } + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + return 0; +} + +static int list_storages(FDFSGroupStat *pGroupStat) +{ + int result; + int storage_count; + FDFSStorageInfo storage_infos[FDFS_MAX_SERVERS_EACH_GROUP]; + FDFSStorageInfo *p; + FDFSStorageInfo *pStorage; + FDFSStorageInfo *pStorageEnd; + FDFSStorageStat *pStorageStat; + char szJoinTime[32]; + char szUpTime[32]; + char szLastHeartBeatTime[32]; + char szSrcUpdTime[32]; + char szSyncUpdTime[32]; + char szSyncedTimestamp[32]; + char szSyncedDelaySeconds[128]; + char szHostname[128]; + char szHostnamePrompt[128+8]; + int k; + int max_last_source_update; + + printf( "group name = %s\n" \ + "disk total space = "INT64_PRINTF_FORMAT" MB\n" \ + "disk free space = "INT64_PRINTF_FORMAT" MB\n" \ + "trunk free space = "INT64_PRINTF_FORMAT" MB\n" \ + "storage server count = %d\n" \ + "active server count = %d\n" \ + "storage server port = %d\n" \ + "storage HTTP port = %d\n" \ + "store path count = %d\n" \ + "subdir count per path = %d\n" \ + "current write server index = %d\n" \ + "current trunk file id = %d\n\n", \ + pGroupStat->group_name, \ + pGroupStat->total_mb, \ + pGroupStat->free_mb, \ + pGroupStat->trunk_free_mb, \ + pGroupStat->count, \ + pGroupStat->active_count, \ + pGroupStat->storage_port, \ + pGroupStat->storage_http_port, \ + pGroupStat->store_path_count, \ + pGroupStat->subdir_count_per_path, \ + pGroupStat->current_write_server, \ + pGroupStat->current_trunk_file_id + ); + + result = tracker_list_servers(pTrackerServer, \ + pGroupStat->group_name, NULL, \ + storage_infos, FDFS_MAX_SERVERS_EACH_GROUP, \ + &storage_count); + if (result != 0) + { + return result; + } + + k = 0; + pStorageEnd = storage_infos + storage_count; + for (pStorage=storage_infos; pStoragestat.last_source_update + > max_last_source_update) + { + max_last_source_update = \ + p->stat.last_source_update; + } + } + + pStorageStat = &(pStorage->stat); + if (max_last_source_update == 0) + { + *szSyncedDelaySeconds = '\0'; + } + else + { + if (pStorageStat->last_synced_timestamp == 0) + { + strcpy(szSyncedDelaySeconds, "(never synced)"); + } + else + { + int delay_seconds; + int remain_seconds; + int day; + int hour; + int minute; + int second; + char szDelayTime[64]; + + delay_seconds = (int)(max_last_source_update - \ + pStorageStat->last_synced_timestamp); + day = delay_seconds / (24 * 3600); + remain_seconds = delay_seconds % (24 * 3600); + hour = remain_seconds / 3600; + remain_seconds %= 3600; + minute = remain_seconds / 60; + second = remain_seconds % 60; + + if (day != 0) + { + sprintf(szDelayTime, "%d days " \ + "%02dh:%02dm:%02ds", \ + day, hour, minute, second); + } + else if (hour != 0) + { + sprintf(szDelayTime, "%02dh:%02dm:%02ds", \ + hour, minute, second); + } + else if (minute != 0) + { + sprintf(szDelayTime, "%02dm:%02ds", minute, second); + } + else + { + sprintf(szDelayTime, "%ds", second); + } + + sprintf(szSyncedDelaySeconds, "(%s delay)", szDelayTime); + } + } + + getHostnameByIp(pStorage->ip_addr, szHostname, sizeof(szHostname)); + if (*szHostname != '\0') + { + sprintf(szHostnamePrompt, " (%s)", szHostname); + } + else + { + *szHostnamePrompt = '\0'; + } + + if (pStorage->up_time != 0) + { + formatDatetime(pStorage->up_time, \ + "%Y-%m-%d %H:%M:%S", \ + szUpTime, sizeof(szUpTime)); + } + else + { + *szUpTime = '\0'; + } + + printf( "\tStorage %d:\n" \ + "\t\tid = %s\n" \ + "\t\tip_addr = %s%s %s\n" \ + "\t\thttp domain = %s\n" \ + "\t\tversion = %s\n" \ + "\t\tjoin time = %s\n" \ + "\t\tup time = %s\n" \ + "\t\ttotal storage = %d MB\n" \ + "\t\tfree storage = %d MB\n" \ + "\t\tupload priority = %d\n" \ + "\t\tstore_path_count = %d\n" \ + "\t\tsubdir_count_per_path = %d\n" \ + "\t\tstorage_port = %d\n" \ + "\t\tstorage_http_port = %d\n" \ + "\t\tcurrent_write_path = %d\n" \ + "\t\tsource storage id= %s\n" \ + "\t\tif_trunk_server= %d\n" \ + "\t\ttotal_upload_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_upload_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_append_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_append_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_modify_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_modify_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_truncate_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_truncate_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_set_meta_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_set_meta_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_delete_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_delete_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_download_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_download_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_get_meta_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_get_meta_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_create_link_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_create_link_count = "INT64_PRINTF_FORMAT"\n"\ + "\t\ttotal_delete_link_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_delete_link_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_upload_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_upload_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_append_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_append_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_modify_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_modify_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tstotal_download_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_download_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_sync_in_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_sync_in_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_sync_out_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_sync_out_bytes = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_file_open_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_file_open_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_file_read_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_file_read_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\ttotal_file_write_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tsuccess_file_write_count = "INT64_PRINTF_FORMAT"\n" \ + "\t\tlast_heart_beat_time = %s\n" \ + "\t\tlast_source_update = %s\n" \ + "\t\tlast_sync_update = %s\n" \ + "\t\tlast_synced_timestamp = %s %s\n", \ + ++k, pStorage->id, pStorage->ip_addr, \ + szHostnamePrompt, get_storage_status_caption( \ + pStorage->status), pStorage->domain_name, \ + pStorage->version, \ + formatDatetime(pStorage->join_time, \ + "%Y-%m-%d %H:%M:%S", \ + szJoinTime, sizeof(szJoinTime)), \ + szUpTime, pStorage->total_mb, \ + pStorage->free_mb, \ + pStorage->upload_priority, \ + pStorage->store_path_count, \ + pStorage->subdir_count_per_path, \ + pStorage->storage_port, \ + pStorage->storage_http_port, \ + pStorage->current_write_path, \ + pStorage->src_id, \ + pStorage->if_trunk_server, \ + pStorageStat->total_upload_count, \ + pStorageStat->success_upload_count, \ + pStorageStat->total_append_count, \ + pStorageStat->success_append_count, \ + pStorageStat->total_modify_count, \ + pStorageStat->success_modify_count, \ + pStorageStat->total_truncate_count, \ + pStorageStat->success_truncate_count, \ + pStorageStat->total_set_meta_count, \ + pStorageStat->success_set_meta_count, \ + pStorageStat->total_delete_count, \ + pStorageStat->success_delete_count, \ + pStorageStat->total_download_count, \ + pStorageStat->success_download_count, \ + pStorageStat->total_get_meta_count, \ + pStorageStat->success_get_meta_count, \ + pStorageStat->total_create_link_count, \ + pStorageStat->success_create_link_count, \ + pStorageStat->total_delete_link_count, \ + pStorageStat->success_delete_link_count, \ + pStorageStat->total_upload_bytes, \ + pStorageStat->success_upload_bytes, \ + pStorageStat->total_append_bytes, \ + pStorageStat->success_append_bytes, \ + pStorageStat->total_modify_bytes, \ + pStorageStat->success_modify_bytes, \ + pStorageStat->total_download_bytes, \ + pStorageStat->success_download_bytes, \ + pStorageStat->total_sync_in_bytes, \ + pStorageStat->success_sync_in_bytes, \ + pStorageStat->total_sync_out_bytes, \ + pStorageStat->success_sync_out_bytes, \ + pStorageStat->total_file_open_count, \ + pStorageStat->success_file_open_count, \ + pStorageStat->total_file_read_count, \ + pStorageStat->success_file_read_count, \ + pStorageStat->total_file_write_count, \ + pStorageStat->success_file_write_count, \ + formatDatetime(pStorageStat->last_heart_beat_time, \ + "%Y-%m-%d %H:%M:%S", \ + szLastHeartBeatTime, sizeof(szLastHeartBeatTime)), \ + formatDatetime(pStorageStat->last_source_update, \ + "%Y-%m-%d %H:%M:%S", \ + szSrcUpdTime, sizeof(szSrcUpdTime)), \ + formatDatetime(pStorageStat->last_sync_update, \ + "%Y-%m-%d %H:%M:%S", \ + szSyncUpdTime, sizeof(szSyncUpdTime)), \ + formatDatetime(pStorageStat->last_synced_timestamp, \ + "%Y-%m-%d %H:%M:%S", \ + szSyncedTimestamp, sizeof(szSyncedTimestamp)),\ + szSyncedDelaySeconds); + } + + return 0; +} + +static int list_all_groups(const char *group_name) +{ + int result; + int group_count; + FDFSGroupStat group_stats[FDFS_MAX_GROUPS]; + FDFSGroupStat *pGroupStat; + FDFSGroupStat *pGroupEnd; + int i; + + result = tracker_list_groups(pTrackerServer, \ + group_stats, FDFS_MAX_GROUPS, \ + &group_count); + if (result != 0) + { + tracker_close_all_connections(); + fdfs_client_destroy(); + return result; + } + + pGroupEnd = group_stats + group_count; + if (group_name == NULL) + { + printf("group count: %d\n", group_count); + i = 0; + for (pGroupStat=group_stats; pGroupStatgroup_name, group_name) == 0) + { + list_storages(pGroupStat); + break; + } + } + } + + return 0; +} + diff --git a/client/fdfs_test.c b/client/fdfs_test.c new file mode 100644 index 0000000..64d467d --- /dev/null +++ b/client/fdfs_test.c @@ -0,0 +1,691 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "fdfs_global.h" +#include "base64.h" +#include "sockopt.h" +#include "logger.h" +#include "fdfs_http_shared.h" + +int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \ + const int current_size) +{ + if (arg == NULL) + { + return EINVAL; + } + + if (fwrite(data, current_size, 1, (FILE *)arg) != 1) + { + return errno != 0 ? errno : EIO; + } + + return 0; +} + +int uploadFileCallback(void *arg, const int64_t file_size, int sock) +{ + int64_t total_send_bytes; + char *filename; + + if (arg == NULL) + { + return EINVAL; + } + + filename = (char *)arg; + return tcpsendfile(sock, filename, file_size, \ + g_fdfs_network_timeout, &total_send_bytes); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + ConnectionInfo storageServer; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[256]; + char master_filename[256]; + FDFSMetaData meta_list[32]; + int meta_count; + int i; + FDFSMetaData *pMetaList; + char token[32 + 1]; + char file_id[128]; + char file_url[256]; + char szDatetime[20]; + char szPortPart[16]; + int url_len; + time_t ts; + char *file_buff; + int64_t file_size; + char *operation; + char *meta_buff; + int store_path_index; + FDFSFileInfo file_info; + + printf("This is FastDFS client test program v%d.%02d\n" \ +"\nCopyright (C) 2008, Happy Fish / YuQing\n" \ +"\nFastDFS may be copied only under the terms of the GNU General\n" \ +"Public License V3, which may be found in the FastDFS source kit.\n" \ +"Please visit the FastDFS Home Page http://www.csource.org/ \n" \ +"for more detail.\n\n" \ +, g_fdfs_version.major, g_fdfs_version.minor); + + if (argc < 3) + { + printf("Usage: %s \n" \ + "\toperation: upload, download, getmeta, setmeta, " \ + "delete and query_servers\n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_DEBUG; + + conf_filename = argv[1]; + operation = argv[2]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + pStorageServer = NULL; + *group_name = '\0'; + local_filename = NULL; + if (strcmp(operation, "upload") == 0) + { + int upload_type; + char *prefix_name; + const char *file_ext_name; + char slave_filename[256]; + int slave_filename_len; + + if (argc < 4) + { + printf("Usage: %s upload " \ + " [FILE | BUFF | CALLBACK] \n",\ + argv[0]); + fdfs_client_destroy(); + return EINVAL; + } + + local_filename = argv[3]; + if (argc == 4) + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + else + { + if (strcmp(argv[4], "BUFF") == 0) + { + upload_type = FDFS_UPLOAD_BY_BUFF; + } + else if (strcmp(argv[4], "CALLBACK") == 0) + { + upload_type = FDFS_UPLOAD_BY_CALLBACK; + } + else + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + } + + store_path_index = 0; + + { + ConnectionInfo storageServers[FDFS_MAX_SERVERS_EACH_GROUP]; + ConnectionInfo *pServer; + ConnectionInfo *pServerEnd; + int storage_count; + + if ((result=tracker_query_storage_store_list_without_group( \ + pTrackerServer, storageServers, \ + FDFS_MAX_SERVERS_EACH_GROUP, &storage_count, \ + group_name, &store_path_index)) == 0) + { + printf("tracker_query_storage_store_list_without_group: \n"); + pServerEnd = storageServers + storage_count; + for (pServer=storageServers; pServerip_addr, pServer->port); + } + printf("\n"); + } + } + + if ((result=tracker_query_storage_store(pTrackerServer, \ + &storageServer, group_name, &store_path_index)) != 0) + { + fdfs_client_destroy(); + printf("tracker_query_storage fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + printf("group_name=%s, ip_addr=%s, port=%d\n", \ + group_name, storageServer.ip_addr, \ + storageServer.port); + + if ((pStorageServer=tracker_connect_server(&storageServer, \ + &result)) == NULL) + { + fdfs_client_destroy(); + return result; + } + + memset(&meta_list, 0, sizeof(meta_list)); + meta_count = 0; + strcpy(meta_list[meta_count].name, "ext_name"); + strcpy(meta_list[meta_count].value, "jpg"); + meta_count++; + strcpy(meta_list[meta_count].name, "width"); + strcpy(meta_list[meta_count].value, "160"); + meta_count++; + strcpy(meta_list[meta_count].name, "height"); + strcpy(meta_list[meta_count].value, "80"); + meta_count++; + strcpy(meta_list[meta_count].name, "file_size"); + strcpy(meta_list[meta_count].value, "115120"); + meta_count++; + + file_ext_name = fdfs_get_file_ext_name(local_filename); + *group_name = '\0'; + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_upload_by_filename(pTrackerServer, \ + pStorageServer, store_path_index, \ + local_filename, file_ext_name, \ + meta_list, meta_count, \ + group_name, remote_filename); + + printf("storage_upload_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_upload_by_filebuff(pTrackerServer, \ + pStorageServer, store_path_index, \ + file_content, file_size, file_ext_name, \ + meta_list, meta_count, \ + group_name, remote_filename); + free(file_content); + } + + printf("storage_upload_by_filebuff\n"); + } + else + { + struct stat stat_buf; + + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_by_callback(pTrackerServer, \ + pStorageServer, store_path_index, \ + uploadFileCallback, local_filename, \ + file_size, file_ext_name, \ + meta_list, meta_count, \ + group_name, remote_filename); + } + + printf("storage_upload_by_callback\n"); + } + + if (result != 0) + { + printf("upload file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + if (g_tracker_server_http_port == 80) + { + *szPortPart = '\0'; + } + else + { + sprintf(szPortPart, ":%d", g_tracker_server_http_port); + } + + sprintf(file_id, "%s/%s", group_name, remote_filename); + url_len = sprintf(file_url, "http://%s%s/%s", \ + pStorageServer->ip_addr, szPortPart, file_id); + if (g_anti_steal_token) + { + ts = time(NULL); + fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \ + ts, token); + sprintf(file_url + url_len, "?token=%s&ts=%d", \ + token, (int)ts); + } + + printf("group_name=%s, remote_filename=%s\n", \ + group_name, remote_filename); + + fdfs_get_file_info(group_name, remote_filename, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("example file url: %s\n", file_url); + + strcpy(master_filename, remote_filename); + *remote_filename = '\0'; + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + prefix_name = "_big"; + result = storage_upload_slave_by_filename(pTrackerServer, + NULL, local_filename, master_filename, \ + prefix_name, file_ext_name, \ + meta_list, meta_count, \ + group_name, remote_filename); + + printf("storage_upload_slave_by_filename\n"); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + prefix_name = "1024x1024"; + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_upload_slave_by_filebuff(pTrackerServer, \ + NULL, file_content, file_size, master_filename, + prefix_name, file_ext_name, \ + meta_list, meta_count, \ + group_name, remote_filename); + free(file_content); + } + + printf("storage_upload_slave_by_filebuff\n"); + } + else + { + struct stat stat_buf; + + prefix_name = "-small"; + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_slave_by_callback(pTrackerServer, \ + NULL, uploadFileCallback, local_filename, \ + file_size, master_filename, prefix_name, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); + } + + printf("storage_upload_slave_by_callback\n"); + } + + if (result != 0) + { + printf("upload slave file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + if (g_tracker_server_http_port == 80) + { + *szPortPart = '\0'; + } + else + { + sprintf(szPortPart, ":%d", g_tracker_server_http_port); + } + + sprintf(file_id, "%s/%s", group_name, remote_filename); + url_len = sprintf(file_url, "http://%s%s/%s", \ + pStorageServer->ip_addr, szPortPart, file_id); + if (g_anti_steal_token) + { + ts = time(NULL); + fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \ + ts, token); + sprintf(file_url + url_len, "?token=%s&ts=%d", \ + token, (int)ts); + } + + printf("group_name=%s, remote_filename=%s\n", \ + group_name, remote_filename); + + fdfs_get_file_info(group_name, remote_filename, &file_info); + + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("example file url: %s\n", file_url); + + if (fdfs_gen_slave_filename(master_filename, \ + prefix_name, file_ext_name, \ + slave_filename, &slave_filename_len) == 0) + { + + if (strcmp(remote_filename, slave_filename) != 0) + { + printf("slave_filename=%s\n" \ + "remote_filename=%s\n" \ + "not equal!\n", \ + slave_filename, remote_filename); + } + } + } + else if (strcmp(operation, "download") == 0 || + strcmp(operation, "getmeta") == 0 || + strcmp(operation, "setmeta") == 0 || + strcmp(operation, "query_servers") == 0 || + strcmp(operation, "delete") == 0) + { + if (argc < 5) + { + printf("Usage: %s %s " \ + " \n", \ + argv[0], operation); + fdfs_client_destroy(); + return EINVAL; + } + + snprintf(group_name, sizeof(group_name), "%s", argv[3]); + snprintf(remote_filename, sizeof(remote_filename), \ + "%s", argv[4]); + if (strcmp(operation, "setmeta") == 0 || + strcmp(operation, "delete") == 0) + { + result = tracker_query_storage_update(pTrackerServer, \ + &storageServer, group_name, remote_filename); + } + else if (strcmp(operation, "query_servers") == 0) + { + ConnectionInfo storageServers[FDFS_MAX_SERVERS_EACH_GROUP]; + int server_count; + + result = tracker_query_storage_list(pTrackerServer, \ + storageServers, FDFS_MAX_SERVERS_EACH_GROUP, \ + &server_count, group_name, remote_filename); + + if (result != 0) + { + printf("tracker_query_storage_list fail, "\ + "group_name=%s, filename=%s, " \ + "error no: %d, error info: %s\n", \ + group_name, remote_filename, \ + result, STRERROR(result)); + } + else + { + printf("server list (%d):\n", server_count); + for (i=0; i= 6) + { + local_filename = argv[5]; + if (strcmp(local_filename, "CALLBACK") == 0) + { + FILE *fp; + fp = fopen(local_filename, "wb"); + if (fp == NULL) + { + result = errno != 0 ? errno : EPERM; + printf("open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + local_filename, result, \ + STRERROR(result)); + } + else + { + result = storage_download_file_ex( \ + pTrackerServer, pStorageServer, \ + group_name, remote_filename, 0, 0, \ + writeToFileCallback, fp, &file_size); + fclose(fp); + } + } + else + { + result = storage_download_file_to_file( \ + pTrackerServer, pStorageServer, \ + group_name, remote_filename, \ + local_filename, &file_size); + } + } + else + { + file_buff = NULL; + if ((result=storage_download_file_to_buff( \ + pTrackerServer, pStorageServer, \ + group_name, remote_filename, \ + &file_buff, &file_size)) == 0) + { + local_filename = strrchr( \ + remote_filename, '/'); + if (local_filename != NULL) + { + local_filename++; //skip / + } + else + { + local_filename=remote_filename; + } + + result = writeToFile(local_filename, \ + file_buff, file_size); + + free(file_buff); + } + } + + if (result == 0) + { + printf("download file success, " \ + "file size="INT64_PRINTF_FORMAT", file save to %s\n", \ + file_size, local_filename); + } + else + { + printf("download file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + } + else if (strcmp(operation, "getmeta") == 0) + { + if ((result=storage_get_metadata(pTrackerServer, \ + pStorageServer, group_name, remote_filename, \ + &pMetaList, &meta_count)) == 0) + { + printf("get meta data success, " \ + "meta count=%d\n", meta_count); + for (i=0; i %s " \ + " " \ + " \n" \ + "\top_flag: %c for overwrite, " \ + "%c for merge\n" \ + "\tmetadata_list: name1=value1," \ + "name2=value2,...\n", \ + argv[0], operation, \ + STORAGE_SET_METADATA_FLAG_OVERWRITE, \ + STORAGE_SET_METADATA_FLAG_MERGE); + fdfs_client_destroy(); + return EINVAL; + } + + meta_buff = strdup(argv[6]); + if (meta_buff == NULL) + { + printf("Out of memory!\n"); + return ENOMEM; + } + + pMetaList = fdfs_split_metadata_ex(meta_buff, \ + ',', '=', &meta_count, &result); + if (pMetaList == NULL) + { + printf("Out of memory!\n"); + free(meta_buff); + return ENOMEM; + } + + if ((result=storage_set_metadata(pTrackerServer, \ + NULL, group_name, remote_filename, \ + pMetaList, meta_count, *argv[5])) == 0) + { + printf("set meta data success\n"); + } + else + { + printf("setmeta fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + + free(meta_buff); + free(pMetaList); + } + else if(strcmp(operation, "delete") == 0) + { + if ((result=storage_delete_file(pTrackerServer, \ + NULL, group_name, remote_filename)) == 0) + { + printf("delete file success\n"); + } + else + { + printf("delete file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + } + } + else + { + fdfs_client_destroy(); + printf("invalid operation: %s\n", operation); + return EINVAL; + } + + /* for test only */ + if ((result=fdfs_active_test(pTrackerServer)) != 0) + { + printf("active_test to tracker server %s:%d fail, errno: %d\n", \ + pTrackerServer->ip_addr, pTrackerServer->port, result); + } + + /* for test only */ + if ((result=fdfs_active_test(pStorageServer)) != 0) + { + printf("active_test to storage server %s:%d fail, errno: %d\n", \ + pStorageServer->ip_addr, pStorageServer->port, result); + } + + tracker_disconnect_server_ex(pStorageServer, true); + tracker_disconnect_server_ex(pTrackerServer, true); + + fdfs_client_destroy(); + + return result; +} + diff --git a/client/fdfs_test1.c b/client/fdfs_test1.c new file mode 100644 index 0000000..b9d0720 --- /dev/null +++ b/client/fdfs_test1.c @@ -0,0 +1,658 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "fdfs_global.h" +#include "base64.h" +#include "fdfs_http_shared.h" +#include "sockopt.h" +#include "logger.h" + +int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \ + const int current_size) +{ + if (arg == NULL) + { + return EINVAL; + } + + if (fwrite(data, current_size, 1, (FILE *)arg) != 1) + { + return errno != 0 ? errno : EIO; + } + + return 0; +} + +int uploadFileCallback(void *arg, const int64_t file_size, int sock) +{ + int64_t total_send_bytes; + char *filename; + if (arg == NULL) + { + return EINVAL; + } + + filename = (char *)arg; + return tcpsendfile(sock, filename, file_size, \ + g_fdfs_network_timeout, &total_send_bytes); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + ConnectionInfo storageServer; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSMetaData meta_list[32]; + int meta_count; + int i; + FDFSMetaData *pMetaList; + char token[32 + 1]; + char file_id[128]; + char master_file_id[128]; + char file_url[256]; + char szDatetime[20]; + char szPortPart[16]; + int url_len; + time_t ts; + char *file_buff; + int64_t file_size; + char *operation; + char *meta_buff; + int store_path_index; + FDFSFileInfo file_info; + + printf("This is FastDFS client test program v%d.%02d\n" \ +"\nCopyright (C) 2008, Happy Fish / YuQing\n" \ +"\nFastDFS may be copied only under the terms of the GNU General\n" \ +"Public License V3, which may be found in the FastDFS source kit.\n" \ +"Please visit the FastDFS Home Page http://www.csource.org/ \n" \ +"for more detail.\n\n" \ +, g_fdfs_version.major, g_fdfs_version.minor); + + if (argc < 3) + { + printf("Usage: %s \n" \ + "\toperation: upload, download, getmeta, setmeta, " \ + "delete and query_servers\n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_DEBUG; + + conf_filename = argv[1]; + operation = argv[2]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + local_filename = NULL; + if (strcmp(operation, "upload") == 0) + { + int upload_type; + char *prefix_name; + const char *file_ext_name; + char slave_file_id[256]; + int slave_file_id_len; + + if (argc < 4) + { + printf("Usage: %s upload " \ + " [FILE | BUFF | CALLBACK] \n",\ + argv[0]); + fdfs_client_destroy(); + return EINVAL; + } + + local_filename = argv[3]; + if (argc == 4) + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + else + { + if (strcmp(argv[4], "BUFF") == 0) + { + upload_type = FDFS_UPLOAD_BY_BUFF; + } + else if (strcmp(argv[4], "CALLBACK") == 0) + { + upload_type = FDFS_UPLOAD_BY_CALLBACK; + } + else + { + upload_type = FDFS_UPLOAD_BY_FILE; + } + } + + { + ConnectionInfo storageServers[FDFS_MAX_SERVERS_EACH_GROUP]; + ConnectionInfo *pServer; + ConnectionInfo *pServerEnd; + int storage_count; + + strcpy(group_name, "group1"); + if ((result=tracker_query_storage_store_list_with_group( \ + pTrackerServer, group_name, storageServers, \ + FDFS_MAX_SERVERS_EACH_GROUP, &storage_count, \ + &store_path_index)) == 0) + { + printf("tracker_query_storage_store_list_with_group: \n"); + pServerEnd = storageServers + storage_count; + for (pServer=storageServers; pServerip_addr, \ + pServer->port); + } + printf("\n"); + } + } + + *group_name = '\0'; + if ((result=tracker_query_storage_store(pTrackerServer, \ + &storageServer, group_name, &store_path_index)) != 0) + { + fdfs_client_destroy(); + printf("tracker_query_storage fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + printf("group_name=%s, ip_addr=%s, port=%d\n", \ + group_name, storageServer.ip_addr, \ + storageServer.port); + + if ((pStorageServer=tracker_connect_server(&storageServer, \ + &result)) == NULL) + { + fdfs_client_destroy(); + return result; + } + + memset(&meta_list, 0, sizeof(meta_list)); + meta_count = 0; + strcpy(meta_list[meta_count].name, "ext_name"); + strcpy(meta_list[meta_count].value, "jpg"); + meta_count++; + strcpy(meta_list[meta_count].name, "width"); + strcpy(meta_list[meta_count].value, "160"); + meta_count++; + strcpy(meta_list[meta_count].name, "height"); + strcpy(meta_list[meta_count].value, "80"); + meta_count++; + strcpy(meta_list[meta_count].name, "file_size"); + strcpy(meta_list[meta_count].value, "115120"); + meta_count++; + + file_ext_name = fdfs_get_file_ext_name(local_filename); + strcpy(group_name, ""); + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + printf("storage_upload_by_filename\n"); + result = storage_upload_by_filename1(pTrackerServer, \ + pStorageServer, store_path_index, \ + local_filename, file_ext_name, \ + meta_list, meta_count, \ + group_name, file_id); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + printf("storage_upload_by_filebuff\n"); + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_upload_by_filebuff1(pTrackerServer, \ + pStorageServer, store_path_index, \ + file_content, file_size, file_ext_name, \ + meta_list, meta_count, \ + group_name, file_id); + free(file_content); + } + } + else + { + struct stat stat_buf; + + printf("storage_upload_by_callback\n"); + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_by_callback1(pTrackerServer, \ + pStorageServer, store_path_index, \ + uploadFileCallback, local_filename, \ + file_size, file_ext_name, \ + meta_list, meta_count, \ + group_name, file_id); + } + } + + if (result != 0) + { + printf("upload file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + if (g_tracker_server_http_port == 80) + { + *szPortPart = '\0'; + } + else + { + sprintf(szPortPart, ":%d", g_tracker_server_http_port); + } + + url_len = sprintf(file_url, "http://%s%s/%s", \ + pStorageServer->ip_addr, szPortPart, file_id); + if (g_anti_steal_token) + { + ts = time(NULL); + fdfs_http_gen_token(&g_anti_steal_secret_key, \ + file_id, ts, token); + sprintf(file_url + url_len, "?token=%s&ts=%d", \ + token, (int)ts); + } + + fdfs_get_file_info1(file_id, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("example file url: %s\n", file_url); + + strcpy(master_file_id, file_id); + *file_id = '\0'; + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + prefix_name = "_big"; + printf("storage_upload_slave_by_filename\n"); + result = storage_upload_slave_by_filename1( \ + pTrackerServer, NULL, \ + local_filename, master_file_id, \ + prefix_name, file_ext_name, \ + meta_list, meta_count, file_id); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *file_content; + prefix_name = "1024x1024"; + printf("storage_upload_slave_by_filebuff\n"); + if ((result=getFileContent(local_filename, \ + &file_content, &file_size)) == 0) + { + result = storage_upload_slave_by_filebuff1( \ + pTrackerServer, NULL, file_content, file_size, \ + master_file_id, prefix_name, file_ext_name, \ + meta_list, meta_count, file_id); + free(file_content); + } + } + else + { + struct stat stat_buf; + + prefix_name = "_small"; + printf("storage_upload_slave_by_callback\n"); + if (stat(local_filename, &stat_buf) == 0 && \ + S_ISREG(stat_buf.st_mode)) + { + file_size = stat_buf.st_size; + result = storage_upload_slave_by_callback1( \ + pTrackerServer, NULL, \ + uploadFileCallback, local_filename, \ + file_size, master_file_id, \ + prefix_name, file_ext_name, \ + meta_list, meta_count, file_id); + } + } + + if (result != 0) + { + printf("upload slave file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + tracker_disconnect_server_ex(pStorageServer, true); + fdfs_client_destroy(); + return result; + } + + if (g_tracker_server_http_port == 80) + { + *szPortPart = '\0'; + } + else + { + sprintf(szPortPart, ":%d", g_tracker_server_http_port); + } + url_len = sprintf(file_url, "http://%s%s/%s", \ + pStorageServer->ip_addr, szPortPart, file_id); + if (g_anti_steal_token) + { + ts = time(NULL); + fdfs_http_gen_token(&g_anti_steal_secret_key, \ + file_id, ts, token); + sprintf(file_url + url_len, "?token=%s&ts=%d", \ + token, (int)ts); + } + + fdfs_get_file_info1(file_id, &file_info); + printf("source ip address: %s\n", file_info.source_ip_addr); + printf("file timestamp=%s\n", formatDatetime( + file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \ + szDatetime, sizeof(szDatetime))); + printf("file size="INT64_PRINTF_FORMAT"\n", file_info.file_size); + printf("file crc32=%u\n", file_info.crc32); + printf("example file url: %s\n", file_url); + + if (fdfs_gen_slave_filename(master_file_id, \ + prefix_name, file_ext_name, \ + slave_file_id, &slave_file_id_len) == 0) + { + if (strcmp(file_id, slave_file_id) != 0) + { + printf("slave_file_id=%s\n" \ + "file_id=%s\n" \ + "not equal!\n", \ + slave_file_id, file_id); + } + } + } + else if (strcmp(operation, "download") == 0 || + strcmp(operation, "getmeta") == 0 || + strcmp(operation, "setmeta") == 0 || + strcmp(operation, "query_servers") == 0 || + strcmp(operation, "delete") == 0) + { + if (argc < 4) + { + printf("Usage: %s %s " \ + "\n", \ + argv[0], operation); + fdfs_client_destroy(); + return EINVAL; + } + + snprintf(file_id, sizeof(file_id), "%s", argv[3]); + if (strcmp(operation, "query_servers") == 0) + { + ConnectionInfo storageServers[FDFS_MAX_SERVERS_EACH_GROUP]; + int server_count; + + result = tracker_query_storage_list1(pTrackerServer, \ + storageServers, FDFS_MAX_SERVERS_EACH_GROUP, \ + &server_count, file_id); + + if (result != 0) + { + printf("tracker_query_storage_list1 fail, "\ + "file_id=%s, " \ + "error no: %d, error info: %s\n", \ + file_id, result, STRERROR(result)); + } + else + { + printf("server list (%d):\n", server_count); + for (i=0; i= 5) + { + local_filename = argv[4]; + if (strcmp(local_filename, "CALLBACK") == 0) + { + FILE *fp; + fp = fopen(local_filename, "wb"); + if (fp == NULL) + { + result = errno != 0 ? errno : EPERM; + printf("open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + local_filename, result, \ + STRERROR(result)); + } + else + { + result = storage_download_file_ex1( \ + pTrackerServer, pStorageServer, \ + file_id, 0, 0, \ + writeToFileCallback, fp, &file_size); + fclose(fp); + } + } + else + { + result = storage_download_file_to_file1( \ + pTrackerServer, pStorageServer, \ + file_id, \ + local_filename, &file_size); + } + } + else + { + file_buff = NULL; + if ((result=storage_download_file_to_buff1( \ + pTrackerServer, pStorageServer, \ + file_id, \ + &file_buff, &file_size)) == 0) + { + local_filename = strrchr( \ + file_id, '/'); + if (local_filename != NULL) + { + local_filename++; //skip / + } + else + { + local_filename=file_id; + } + + result = writeToFile(local_filename, \ + file_buff, file_size); + + free(file_buff); + } + } + + if (result == 0) + { + printf("download file success, " \ + "file size="INT64_PRINTF_FORMAT", file save to %s\n", \ + file_size, local_filename); + } + else + { + printf("download file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + } + else if (strcmp(operation, "getmeta") == 0) + { + if ((result=storage_get_metadata1(pTrackerServer, \ + NULL, file_id, \ + &pMetaList, &meta_count)) == 0) + { + printf("get meta data success, " \ + "meta count=%d\n", meta_count); + for (i=0; i %s " \ + " " \ + " \n" \ + "\top_flag: %c for overwrite, " \ + "%c for merge\n" \ + "\tmetadata_list: name1=value1," \ + "name2=value2,...\n", \ + argv[0], operation, \ + STORAGE_SET_METADATA_FLAG_OVERWRITE, \ + STORAGE_SET_METADATA_FLAG_MERGE); + fdfs_client_destroy(); + return EINVAL; + } + + meta_buff = strdup(argv[5]); + if (meta_buff == NULL) + { + printf("Out of memory!\n"); + return ENOMEM; + } + + pMetaList = fdfs_split_metadata_ex(meta_buff, \ + ',', '=', &meta_count, &result); + if (pMetaList == NULL) + { + printf("Out of memory!\n"); + free(meta_buff); + return ENOMEM; + } + + if ((result=storage_set_metadata1(pTrackerServer, \ + NULL, file_id, \ + pMetaList, meta_count, *argv[4])) == 0) + { + printf("set meta data success\n"); + } + else + { + printf("setmeta fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + + free(meta_buff); + free(pMetaList); + } + else if(strcmp(operation, "delete") == 0) + { + if ((result=storage_delete_file1(pTrackerServer, \ + NULL, file_id)) == 0) + { + printf("delete file success\n"); + } + else + { + printf("delete file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + } + } + else + { + fdfs_client_destroy(); + printf("invalid operation: %s\n", operation); + return EINVAL; + } + + /* for test only */ + if ((result=fdfs_active_test(pTrackerServer)) != 0) + { + printf("active_test to tracker server %s:%d fail, errno: %d\n", \ + pTrackerServer->ip_addr, pTrackerServer->port, result); + } + + /* for test only */ + if ((result=fdfs_active_test(pStorageServer)) != 0) + { + printf("active_test to storage server %s:%d fail, errno: %d\n", \ + pStorageServer->ip_addr, pStorageServer->port, result); + } + + tracker_disconnect_server_ex(pStorageServer, true); + tracker_disconnect_server_ex(pTrackerServer, true); + + fdfs_client_destroy(); + + return result; +} + diff --git a/client/fdfs_upload_appender.c b/client/fdfs_upload_appender.c new file mode 100644 index 0000000..ebd02b7 --- /dev/null +++ b/client/fdfs_upload_appender.c @@ -0,0 +1,88 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + ConnectionInfo *pTrackerServer; + int result; + int store_path_index; + ConnectionInfo storageServer; + char file_id[128]; + + if (argc < 3) + { + printf("Usage: %s \n", argv[0]); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_ERR; + ignore_signal_pipe(); + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + + *group_name = '\0'; + store_path_index = 0; + if ((result=tracker_query_storage_store(pTrackerServer, \ + &storageServer, group_name, &store_path_index)) != 0) + { + fdfs_client_destroy(); + fprintf(stderr, "tracker_query_storage fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + local_filename = argv[2]; + result = storage_upload_appender_by_filename1(pTrackerServer, \ + &storageServer, store_path_index, \ + local_filename, NULL, \ + NULL, 0, group_name, file_id); + if (result != 0) + { + fprintf(stderr, "upload file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + return result; + } + + printf("%s\n", file_id); + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + + return 0; +} + diff --git a/client/fdfs_upload_file.c b/client/fdfs_upload_file.c new file mode 100644 index 0000000..4ac119f --- /dev/null +++ b/client/fdfs_upload_file.c @@ -0,0 +1,120 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" + +static void usage(char *argv[]) +{ + printf("Usage: %s " \ + "[storage_ip:port] [store_path_index]\n", argv[0]); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + char *local_filename; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + ConnectionInfo *pTrackerServer; + int result; + int store_path_index; + ConnectionInfo storageServer; + char file_id[128]; + + if (argc < 3) + { + usage(argv); + return 1; + } + + log_init(); + g_log_context.log_level = LOG_ERR; + ignore_signal_pipe(); + + conf_filename = argv[1]; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + fdfs_client_destroy(); + return errno != 0 ? errno : ECONNREFUSED; + } + + local_filename = argv[2]; + *group_name = '\0'; + if (argc >= 4) + { + const char *pPort; + const char *pIpAndPort; + + pIpAndPort = argv[3]; + pPort = strchr(pIpAndPort, ':'); + if (pPort == NULL) + { + fdfs_client_destroy(); + fprintf(stderr, "invalid storage ip address and " \ + "port: %s\n", pIpAndPort); + usage(argv); + return 1; + } + + storageServer.sock = -1; + snprintf(storageServer.ip_addr, sizeof(storageServer.ip_addr), \ + "%.*s", (int)(pPort - pIpAndPort), pIpAndPort); + storageServer.port = atoi(pPort + 1); + if (argc >= 5) + { + store_path_index = atoi(argv[4]); + } + else + { + store_path_index = -1; + } + } + else if ((result=tracker_query_storage_store(pTrackerServer, \ + &storageServer, group_name, &store_path_index)) != 0) + { + fdfs_client_destroy(); + fprintf(stderr, "tracker_query_storage fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + return result; + } + + result = storage_upload_by_filename1(pTrackerServer, \ + &storageServer, store_path_index, \ + local_filename, NULL, \ + NULL, 0, group_name, file_id); + if (result == 0) + { + printf("%s\n", file_id); + } + else + { + fprintf(stderr, "upload file fail, " \ + "error no: %d, error info: %s\n", \ + result, STRERROR(result)); + } + + tracker_disconnect_server_ex(pTrackerServer, true); + fdfs_client_destroy(); + + return result; +} + diff --git a/client/storage_client.c b/client/storage_client.c new file mode 100644 index 0000000..aabd37a --- /dev/null +++ b/client/storage_client.c @@ -0,0 +1,2315 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "client_func.h" +#include "tracker_client.h" +#include "storage_client.h" +#include "storage_client1.h" +#include "client_global.h" +#include "base64.h" + +static struct base64_context the_base64_context; +static int the_base64_context_inited = 0; + +#define FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) \ + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; \ + char *group_name; \ + char *filename; \ + char *pSeperator; \ + \ + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); \ + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); \ + if (pSeperator == NULL) \ + { \ + return EINVAL; \ + } \ + \ + *pSeperator = '\0'; \ + group_name = new_file_id; \ + filename = pSeperator + 1; \ + +#define storage_get_read_connection(pTrackerServer, \ + ppStorageServer, group_name, filename, \ + pNewStorage, new_connection) \ + storage_get_connection(pTrackerServer, \ + ppStorageServer, TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE, \ + group_name, filename, pNewStorage, new_connection) + +#define storage_get_update_connection(pTrackerServer, \ + ppStorageServer, group_name, filename, \ + pNewStorage, new_connection) \ + storage_get_connection(pTrackerServer, \ + ppStorageServer, TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE, \ + group_name, filename, pNewStorage, new_connection) + +static int storage_get_connection(ConnectionInfo *pTrackerServer, \ + ConnectionInfo **ppStorageServer, const byte cmd, \ + const char *group_name, const char *filename, \ + ConnectionInfo *pNewStorage, bool *new_connection) +{ + int result; + bool new_tracker_connection; + ConnectionInfo *pNewTracker; + if (*ppStorageServer == NULL) + { + CHECK_CONNECTION(pTrackerServer, pNewTracker, result, \ + new_tracker_connection); + if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE) + { + result = tracker_query_storage_fetch(pNewTracker, \ + pNewStorage, group_name, filename); + } + else + { + result = tracker_query_storage_update(pNewTracker, \ + pNewStorage, group_name, filename); + } + + if (new_tracker_connection) + { + tracker_disconnect_server_ex(pNewTracker, result != 0); + } + + if (result != 0) + { + return result; + } + + if ((*ppStorageServer=tracker_connect_server(pNewStorage, \ + &result)) == NULL) + { + return result; + } + + *new_connection = true; + } + else + { + if ((*ppStorageServer)->sock >= 0) + { + *new_connection = false; + } + else + { + if ((*ppStorageServer=tracker_connect_server( \ + *ppStorageServer, &result)) == NULL) + { + return result; + } + + *new_connection = true; + } + } + + return 0; +} + +static int storage_get_upload_connection(ConnectionInfo *pTrackerServer, \ + ConnectionInfo **ppStorageServer, char *group_name, \ + ConnectionInfo *pNewStorage, int *store_path_index, \ + bool *new_connection) +{ + int result; + bool new_tracker_connection; + ConnectionInfo *pNewTracker; + + if (*ppStorageServer == NULL) + { + CHECK_CONNECTION(pTrackerServer, pNewTracker, result, \ + new_tracker_connection); + if (*group_name == '\0') + { + result = tracker_query_storage_store_without_group( \ + pNewTracker, pNewStorage, group_name, \ + store_path_index); + } + else + { + result = tracker_query_storage_store_with_group( \ + pNewTracker, group_name, pNewStorage, \ + store_path_index); + } + + if (new_tracker_connection) + { + tracker_disconnect_server_ex(pNewTracker, result != 0); + } + + if (result != 0) + { + return result; + } + + if ((*ppStorageServer=tracker_connect_server(pNewStorage, \ + &result)) == NULL) + { + return result; + } + + *new_connection = true; + } + else + { + if ((*ppStorageServer)->sock >= 0) + { + *new_connection = false; + } + else + { + if ((*ppStorageServer=tracker_connect_server( \ + *ppStorageServer, &result)) == NULL) + { + return result; + } + + *new_connection = true; + } + } + + return 0; +} + +int storage_get_metadata1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + FDFSMetaData **meta_list, int *meta_count) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return storage_get_metadata(pTrackerServer, pStorageServer, \ + group_name, filename, meta_list, meta_count); +} + +int storage_get_metadata(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename, \ + FDFSMetaData **meta_list, \ + int *meta_count) +{ + TrackerHeader *pHeader; + int result; + ConnectionInfo storageServer; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+128]; + int64_t in_bytes; + int filename_len; + char *file_buff; + int64_t file_size; + bool new_connection; + + file_buff = NULL; + *meta_list = NULL; + *meta_count = 0; + + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + do + { + /** + send pkg format: + FDFS_GROUP_NAME_MAX_LEN bytes: group_name + remain bytes: filename + **/ + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + filename_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", filename); + + long2buff(FDFS_GROUP_NAME_MAX_LEN + filename_len, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_GET_METADATA; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + filename_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + if ((result=fdfs_recv_response(pStorageServer, \ + &file_buff, 0, &in_bytes)) != 0) + { + break; + } + + file_size = in_bytes; + if (file_size == 0) + { + break; + } + + file_buff[in_bytes] = '\0'; + *meta_list = fdfs_split_metadata(file_buff, meta_count, &result); + } while (0); + + if (file_buff != NULL) + { + free(file_buff); + } + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_query_file_info_ex1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_id, \ + FDFSFileInfo *pFileInfo, const bool bSilence) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + return storage_query_file_info_ex(pTrackerServer, pStorageServer, \ + group_name, filename, pFileInfo, bSilence); +} + +int storage_query_file_info_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename, \ + FDFSFileInfo *pFileInfo, const bool bSilence) +{ + TrackerHeader *pHeader; + int result; + ConnectionInfo storageServer; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+128]; + char in_buff[3 * FDFS_PROTO_PKG_LEN_SIZE + IP_ADDRESS_SIZE]; + char buff[64]; + int64_t in_bytes; + int filename_len; + int buff_len; + char *pInBuff; + char *p; + bool new_connection; + + if ((result=storage_get_read_connection(pTrackerServer, \ + &pStorageServer, group_name, filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + do + { + /** + send pkg format: + FDFS_GROUP_NAME_MAX_LEN bytes: group_name + remain bytes: filename + **/ + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + filename_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", filename); + + long2buff(FDFS_GROUP_NAME_MAX_LEN + filename_len, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_QUERY_FILE_INFO; + pHeader->status = bSilence ? ENOENT : 0; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + filename_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + pInBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pInBuff, sizeof(in_buff), &in_bytes)) != 0) + { + break; + } + + if (in_bytes != sizeof(in_buff)) + { + logError("file: "__FILE__", line: %d, " \ + "recv data from storage server %s:%d fail, " \ + "recv bytes: "INT64_PRINTF_FORMAT" != %d", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + in_bytes, (int)sizeof(in_buff)); + result = EINVAL; + } + + if (!the_base64_context_inited) + { + the_base64_context_inited = 1; + base64_init_ex(&the_base64_context, 0, '-', '_', '.'); + } + + memset(buff, 0, sizeof(buff)); + if (filename_len >= FDFS_LOGIC_FILE_PATH_LEN \ + + FDFS_FILENAME_BASE64_LENGTH + FDFS_FILE_EXT_NAME_MAX_LEN + 1) + { + base64_decode_auto(&the_base64_context, (char *)filename + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + buff, &buff_len); + } + + p = in_buff; + pFileInfo->file_size = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + pFileInfo->create_timestamp = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + pFileInfo->crc32 = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + memcpy(pFileInfo->source_ip_addr, p, IP_ADDRESS_SIZE); + *(pFileInfo->source_ip_addr + IP_ADDRESS_SIZE - 1) = '\0'; + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_delete_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return storage_delete_file(pTrackerServer, \ + pStorageServer, group_name, filename); +} + +int storage_truncate_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, + const char *appender_file_id, \ + const int64_t truncated_file_size) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_truncate_file(pTrackerServer, \ + pStorageServer, group_name, filename, \ + truncated_file_size); +} + +int storage_delete_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename) +{ + TrackerHeader *pHeader; + int result; + ConnectionInfo storageServer; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+128]; + char in_buff[1]; + char *pBuff; + int64_t in_bytes; + int filename_len; + bool new_connection; + + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + do + { + /** + send pkg format: + FDFS_GROUP_NAME_MAX_LEN bytes: group_name + remain bytes: filename + **/ + + memset(out_buff, 0, sizeof(out_buff)); + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + filename_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", filename); + + pHeader = (TrackerHeader *)out_buff; + long2buff(FDFS_GROUP_NAME_MAX_LEN + filename_len, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_DELETE_FILE; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + filename_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + pBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pBuff, 0, &in_bytes)) != 0) + { + break; + } + + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_do_download_file1_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const int download_type, const char *file_id, \ + const int64_t file_offset, const int64_t download_bytes, \ + char **file_buff, void *arg, int64_t *file_size) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + download_type, group_name, filename, \ + file_offset, download_bytes, file_buff, arg, file_size); +} + +int storage_do_download_file_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const int download_type, \ + const char *group_name, const char *remote_filename, \ + const int64_t file_offset, const int64_t download_bytes, \ + char **file_buff, void *arg, int64_t *file_size) +{ + TrackerHeader *pHeader; + int result; + ConnectionInfo storageServer; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+128]; + char *p; + int out_bytes; + int64_t in_bytes; + int64_t total_recv_bytes; + int filename_len; + bool new_connection; + + *file_size = 0; + if ((result=storage_get_read_connection(pTrackerServer, \ + &pStorageServer, group_name, remote_filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + do + { + /** + send pkg format: + 8 bytes: file offset + 8 bytes: download file bytes + FDFS_GROUP_NAME_MAX_LEN bytes: group_name + remain bytes: filename + **/ + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + long2buff(file_offset, p); + p += 8; + long2buff(download_bytes, p); + p += 8; + snprintf(p, sizeof(out_buff) - (p - out_buff), "%s", group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + filename_len = snprintf(p, sizeof(out_buff) - (p - out_buff), \ + "%s", remote_filename); + p += filename_len; + out_bytes = p - out_buff; + long2buff(out_bytes - sizeof(TrackerHeader), pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_DOWNLOAD_FILE; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + out_bytes, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + if (download_type == FDFS_DOWNLOAD_TO_FILE) + { + if ((result=fdfs_recv_header(pStorageServer, \ + &in_bytes)) != 0) + { + break; + } + + if ((result=tcprecvfile(pStorageServer->sock, \ + *file_buff, in_bytes, 0, \ + g_fdfs_network_timeout, \ + &total_recv_bytes)) != 0) + { + break; + } + } + else if (download_type == FDFS_DOWNLOAD_TO_BUFF) + { + *file_buff = NULL; + if ((result=fdfs_recv_response(pStorageServer, \ + file_buff, 0, &in_bytes)) != 0) + { + break; + } + } + else + { + DownloadCallback callback; + char buff[2048]; + int recv_bytes; + int64_t remain_bytes; + + if ((result=fdfs_recv_header(pStorageServer, \ + &in_bytes)) != 0) + { + break; + } + + callback = (DownloadCallback)*file_buff; + remain_bytes = in_bytes; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + recv_bytes = sizeof(buff); + } + else + { + recv_bytes = remain_bytes; + } + + if ((result=tcprecvdata_nb(pStorageServer->sock, buff, \ + recv_bytes, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "recv data from storage server " \ + "%s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + result = callback(arg, in_bytes, buff, recv_bytes); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call callback function fail, " \ + "error code: %d", __LINE__, result); + break; + } + + remain_bytes -= recv_bytes; + } + + if (remain_bytes != 0) + { + break; + } + } + + *file_size = in_bytes; + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_download_file_to_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + const char *local_filename, int64_t *file_size) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return storage_download_file_to_file(pTrackerServer, \ + pStorageServer, group_name, filename, \ + local_filename, file_size); +} + +int storage_download_file_to_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *remote_filename, \ + const char *local_filename, int64_t *file_size) +{ + char *pLocalFilename; + pLocalFilename = (char *)local_filename; + return storage_do_download_file(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_FILE, group_name, remote_filename, \ + &pLocalFilename, NULL, file_size); +} + +int storage_upload_by_filename1_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const char *local_filename, \ + const char *file_ext_name, const FDFSMetaData *meta_list, \ + const int meta_count, const char *group_name, char *file_id) +{ + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + int result; + + if (group_name == NULL) + { + *new_group_name = '\0'; + } + else + { + snprintf(new_group_name, sizeof(new_group_name), \ + "%s", group_name); + } + + result = storage_upload_by_filename_ex(pTrackerServer, \ + pStorageServer, store_path_index, cmd, \ + local_filename, file_ext_name, \ + meta_list, meta_count, \ + new_group_name, remote_filename); + if (result == 0) + { + sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + } + else + { + file_id[0] = '\0'; + } + + return result; +} + +int storage_do_upload_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const int upload_type, \ + const char *file_buff, void *arg, const int64_t file_size, \ + const char *file_ext_name, const FDFSMetaData *meta_list, \ + const int meta_count, const char *group_name, char *file_id) +{ + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + int result; + + if (group_name == NULL) + { + *new_group_name = '\0'; + } + else + { + snprintf(new_group_name, sizeof(new_group_name), \ + "%s", group_name); + } + + result = storage_do_upload_file(pTrackerServer, \ + pStorageServer, store_path_index, cmd, upload_type, \ + file_buff, arg, file_size, NULL, NULL, file_ext_name, \ + meta_list, meta_count, new_group_name, remote_filename); + if (result == 0) + { + sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + } + else + { + file_id[0] = '\0'; + } + + return result; +} + +/** +STORAGE_PROTO_CMD_UPLOAD_FILE and +STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE: +1 byte: store path index +8 bytes: meta data bytes +8 bytes: file size +FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name +meta data bytes: each meta data seperated by \x01, + name and value seperated by \x02 +file size bytes: file content + +STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE: +8 bytes: master filename length +8 bytes: meta data bytes +8 bytes: file size +FDFS_FILE_PREFIX_MAX_LEN bytes : filename prefix +FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.) +master filename bytes: master filename +meta data bytes: each meta data seperated by \x01, + name and value seperated by \x02 +file size bytes: file content +**/ +int storage_do_upload_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const int upload_type, const char *file_buff, \ + void *arg, const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename) +{ + TrackerHeader *pHeader; + int result; + char out_buff[512]; + char *p; + int64_t in_bytes; + int64_t total_send_bytes; + char in_buff[128]; + char *pInBuff; + ConnectionInfo storageServer; + bool new_connection; + bool bUploadSlave; + int new_store_path; + int master_filename_len; + int prefix_len; + + *remote_filename = '\0'; + new_store_path = store_path_index; + if (master_filename != NULL) + { + master_filename_len = strlen(master_filename); + } + else + { + master_filename_len = 0; + } + + if (prefix_name != NULL) + { + prefix_len = strlen(prefix_name); + } + else + { + prefix_len = 0; + } + + bUploadSlave = (strlen(group_name) > 0 && master_filename_len > 0); + if (bUploadSlave) + { + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, master_filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + } + else if ((result=storage_get_upload_connection(pTrackerServer, \ + &pStorageServer, group_name, &storageServer, \ + &new_store_path, &new_connection)) != 0) + { + *group_name = '\0'; + return result; + } + + *group_name = '\0'; + + /* + //logInfo("upload to storage %s:%d\n", \ + pStorageServer->ip_addr, pStorageServer->port); + */ + + do + { + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + if (bUploadSlave) + { + long2buff(master_filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + } + else + { + *p++ = (char)new_store_path; + } + + long2buff(file_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + if (bUploadSlave) + { + memset(p, 0, FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN); + if (prefix_len > FDFS_FILE_PREFIX_MAX_LEN) + { + prefix_len = FDFS_FILE_PREFIX_MAX_LEN; + } + if (prefix_len > 0) + { + memcpy(p, prefix_name, prefix_len); + } + p += FDFS_FILE_PREFIX_MAX_LEN; + } + else + { + memset(p, 0, FDFS_FILE_EXT_NAME_MAX_LEN); + } + + if (file_ext_name != NULL) + { + int file_ext_len; + + file_ext_len = strlen(file_ext_name); + if (file_ext_len > FDFS_FILE_EXT_NAME_MAX_LEN) + { + file_ext_len = FDFS_FILE_EXT_NAME_MAX_LEN; + } + if (file_ext_len > 0) + { + memcpy(p, file_ext_name, file_ext_len); + } + } + p += FDFS_FILE_EXT_NAME_MAX_LEN; + + if (bUploadSlave) + { + memcpy(p, master_filename, master_filename_len); + p += master_filename_len; + } + + long2buff((p - out_buff) + file_size - sizeof(TrackerHeader), \ + pHeader->pkg_len); + pHeader->cmd = cmd; + pHeader->status = 0; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + if ((result=tcpsendfile(pStorageServer->sock, file_buff, \ + file_size, g_fdfs_network_timeout, \ + &total_send_bytes)) != 0) + { + break; + } + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + if ((result=tcpsenddata_nb(pStorageServer->sock, \ + (char *)file_buff, file_size, \ + g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + } + else //FDFS_UPLOAD_BY_CALLBACK + { + UploadCallback callback; + callback = (UploadCallback)file_buff; + if ((result=callback(arg, file_size, pStorageServer->sock))!=0) + { + break; + } + } + + pInBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pInBuff, sizeof(in_buff), &in_bytes)) != 0) + { + break; + } + + if (in_bytes <= FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "should > %d", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + in_bytes, FDFS_GROUP_NAME_MAX_LEN); + result = EINVAL; + break; + } + + in_buff[in_bytes] = '\0'; + memcpy(group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0'; + + memcpy(remote_filename, in_buff + FDFS_GROUP_NAME_MAX_LEN, \ + in_bytes - FDFS_GROUP_NAME_MAX_LEN + 1); + + } while (0); + + if (result == 0 && meta_count > 0) + { + result = storage_set_metadata(pTrackerServer, \ + pStorageServer, group_name, remote_filename, \ + meta_list, meta_count, \ + STORAGE_SET_METADATA_FLAG_OVERWRITE); + if (result != 0) //rollback + { + storage_delete_file(pTrackerServer, pStorageServer, \ + group_name, remote_filename); + *group_name = '\0'; + *remote_filename = '\0'; + } + } + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_upload_by_callback_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, UploadCallback callback, void *arg, \ + const int64_t file_size, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename) +{ + return storage_do_upload_file(pTrackerServer, pStorageServer, \ + store_path_index, cmd, FDFS_UPLOAD_BY_CALLBACK, \ + (char *)callback, arg, file_size, NULL, NULL, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); +} + +int storage_upload_by_callback1_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, UploadCallback callback, void *arg, \ + const int64_t file_size, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + const char *group_name, char *file_id) +{ + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + int result; + + if (group_name == NULL) + { + *new_group_name = '\0'; + } + else + { + snprintf(new_group_name, sizeof(new_group_name), \ + "%s", group_name); + } + + result = storage_do_upload_file(pTrackerServer, \ + pStorageServer, store_path_index, \ + cmd, FDFS_UPLOAD_BY_CALLBACK, (char *)callback, arg, \ + file_size, NULL, NULL, file_ext_name, \ + meta_list, meta_count, \ + new_group_name, remote_filename); + if (result == 0) + { + sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + } + else + { + file_id[0] = '\0'; + } + + return result; +} + +int storage_upload_by_filename_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const char *local_filename, \ + const char *file_ext_name, const FDFSMetaData *meta_list, \ + const int meta_count, char *group_name, char *remote_filename) +{ + struct stat stat_buf; + + if (stat(local_filename, &stat_buf) != 0) + { + group_name[0] = '\0'; + remote_filename[0] = '\0'; + return errno; + } + + if (!S_ISREG(stat_buf.st_mode)) + { + group_name[0] = '\0'; + remote_filename[0] = '\0'; + return EINVAL; + } + + if (file_ext_name == NULL) + { + file_ext_name = fdfs_get_file_ext_name(local_filename); + } + + return storage_do_upload_file(pTrackerServer, pStorageServer, \ + store_path_index, cmd, \ + FDFS_UPLOAD_BY_FILE, local_filename, \ + NULL, stat_buf.st_size, NULL, NULL, file_ext_name, \ + meta_list, meta_count, group_name, remote_filename); +} + +int storage_set_metadata1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + const FDFSMetaData *meta_list, const int meta_count, \ + const char op_flag) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return storage_set_metadata(pTrackerServer, pStorageServer, \ + group_name, filename, \ + meta_list, meta_count, op_flag); +} + +/** +8 bytes: filename length +8 bytes: meta data size +1 bytes: operation flag, + 'O' for overwrite all old metadata + 'M' for merge, insert when the meta item not exist, otherwise update it +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +meta data bytes: each meta data seperated by \x01, + name and value seperated by \x02 +**/ +int storage_set_metadata(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename, \ + const FDFSMetaData *meta_list, const int meta_count, \ + const char op_flag) +{ + TrackerHeader *pHeader; + int result; + ConnectionInfo storageServer; + char out_buff[sizeof(TrackerHeader)+2*FDFS_PROTO_PKG_LEN_SIZE+\ + FDFS_GROUP_NAME_MAX_LEN+128]; + char in_buff[1]; + int64_t in_bytes; + char *pBuff; + int filename_len; + char *meta_buff; + int meta_bytes; + char *p; + char *pEnd; + bool new_connection; + + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + meta_buff = NULL; + do + { + memset(out_buff, 0, sizeof(out_buff)); + filename_len = strlen(filename); + + if (meta_count > 0) + { + meta_buff = fdfs_pack_metadata(meta_list, meta_count, \ + NULL, &meta_bytes); + if (meta_buff == NULL) + { + result = ENOMEM; + break; + } + } + else + { + meta_bytes = 0; + } + + pEnd = out_buff + sizeof(out_buff); + p = out_buff + sizeof(TrackerHeader); + + long2buff(filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(meta_bytes, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + *p++ = op_flag; + + snprintf(p, pEnd - p, "%s", group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + + filename_len = snprintf(p, pEnd - p, "%s", filename); + p += filename_len; + + pHeader = (TrackerHeader *)out_buff; + long2buff((int)(p - (out_buff + sizeof(TrackerHeader))) + \ + meta_bytes, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_SET_METADATA; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + if (meta_bytes > 0 && (result=tcpsenddata_nb(pStorageServer->sock, \ + meta_buff, meta_bytes, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + pBuff = in_buff; + result = fdfs_recv_response(pStorageServer, \ + &pBuff, 0, &in_bytes); + } while (0); + + if (meta_buff != NULL) + { + free(meta_buff); + } + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_download_file_ex1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + const int64_t file_offset, const int64_t download_bytes, \ + DownloadCallback callback, void *arg, int64_t *file_size) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return storage_download_file_ex(pTrackerServer, pStorageServer, \ + group_name, filename, file_offset, download_bytes, \ + callback, arg, file_size); +} + +int storage_download_file_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *remote_filename, \ + const int64_t file_offset, const int64_t download_bytes, \ + DownloadCallback callback, void *arg, int64_t *file_size) +{ + char *pCallback; + pCallback = (char *)callback; + return storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_CALLBACK, group_name, remote_filename, \ + file_offset, download_bytes, &pCallback, arg, file_size); +} + +int tracker_query_storage_fetch1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return tracker_query_storage_fetch(pTrackerServer, \ + pStorageServer, group_name, filename); +} + +int tracker_query_storage_update1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return tracker_query_storage_update(pTrackerServer, \ + pStorageServer, group_name, filename); +} + +/** +pkg format: +Header +8 bytes: master filename len +8 bytes: source filename len +8 bytes: source file signature len +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +FDFS_FILE_PREFIX_MAX_LEN bytes : filename prefix, can be empty +FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.) +master filename len: master filename +source filename len: source filename without group name +source file signature len: source file signature +**/ +int storage_client_create_link(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *master_filename,\ + const char *src_filename, const int src_filename_len, \ + const char *src_file_sig, const int src_file_sig_len, \ + const char *group_name, const char *prefix_name, \ + const char *file_ext_name, \ + char *remote_filename, int *filename_len) +{ + TrackerHeader *pHeader; + int result; + char out_buff[sizeof(TrackerHeader) + 4 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN + 256]; + char in_buff[128]; + char *p; + int group_name_len; + int master_filename_len; + int64_t in_bytes; + char *pInBuff; + ConnectionInfo storageServer; + bool new_connection; + + *remote_filename = '\0'; + if (master_filename != NULL) + { + master_filename_len = strlen(master_filename); + } + else + { + master_filename_len = 0; + } + if (src_filename_len >= 128 || src_file_sig_len > 64 || \ + master_filename_len >= 128) + { + return EINVAL; + } + + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, src_filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + do + { + memset(out_buff, 0, sizeof(out_buff)); + p = out_buff + sizeof(TrackerHeader); + long2buff(master_filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + long2buff(src_filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + long2buff(src_file_sig_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + group_name_len = strlen(group_name); + if (group_name_len > FDFS_GROUP_NAME_MAX_LEN) + { + group_name_len = FDFS_GROUP_NAME_MAX_LEN; + } + memcpy(p, group_name, group_name_len); + p += FDFS_GROUP_NAME_MAX_LEN; + + if (prefix_name != NULL) + { + int prefix_len; + + prefix_len = strlen(prefix_name); + if (prefix_len > FDFS_FILE_PREFIX_MAX_LEN) + { + prefix_len = FDFS_FILE_PREFIX_MAX_LEN; + } + if (prefix_len > 0) + { + memcpy(p, prefix_name, prefix_len); + } + } + p += FDFS_FILE_PREFIX_MAX_LEN; + + if (file_ext_name != NULL) + { + int file_ext_len; + + file_ext_len = strlen(file_ext_name); + if (file_ext_len > FDFS_FILE_EXT_NAME_MAX_LEN) + { + file_ext_len = FDFS_FILE_EXT_NAME_MAX_LEN; + } + if (file_ext_len > 0) + { + memcpy(p, file_ext_name, file_ext_len); + } + } + p += FDFS_FILE_EXT_NAME_MAX_LEN; + + if (master_filename_len > 0) + { + memcpy(p, master_filename, master_filename_len); + p += master_filename_len; + } + memcpy(p, src_filename, src_filename_len); + p += src_filename_len; + memcpy(p, src_file_sig, src_file_sig_len); + p += src_file_sig_len; + + pHeader = (TrackerHeader *)out_buff; + long2buff(p - out_buff - sizeof(TrackerHeader), pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_CREATE_LINK; + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + pInBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pInBuff, sizeof(in_buff), &in_bytes)) != 0) + { + break; + } + + if (in_bytes <= FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "should > %d", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + in_bytes, FDFS_GROUP_NAME_MAX_LEN); + result = EINVAL; + break; + } + + *(in_buff + in_bytes) = '\0'; + *filename_len = in_bytes - FDFS_GROUP_NAME_MAX_LEN; + memcpy(remote_filename, in_buff + FDFS_GROUP_NAME_MAX_LEN, \ + (*filename_len) + 1); + + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int tracker_query_storage_list1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int nMaxServerCount, \ + int *server_count, const char *file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + return tracker_query_storage_list(pTrackerServer, \ + pStorageServer, nMaxServerCount, \ + server_count, group_name, filename); +} + +int storage_upload_slave_by_filename(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *master_filename, const char *prefix_name, \ + const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename) +{ + struct stat stat_buf; + + if (master_filename == NULL || *master_filename == '\0' || \ + prefix_name == NULL || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + if (stat(local_filename, &stat_buf) != 0) + { + *group_name = '\0'; + *remote_filename = '\0'; + return errno != 0 ? errno : EPERM; + } + + if (!S_ISREG(stat_buf.st_mode)) + { + *group_name = '\0'; + *remote_filename = '\0'; + return EINVAL; + } + + if (file_ext_name == NULL) + { + file_ext_name = fdfs_get_file_ext_name(local_filename); + } + + return storage_do_upload_file(pTrackerServer, pStorageServer, \ + 0, STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, \ + FDFS_UPLOAD_BY_FILE, local_filename, \ + NULL, stat_buf.st_size, master_filename, prefix_name, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); +} + +int storage_upload_slave_by_callback(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename) +{ + if (master_filename == NULL || *master_filename == '\0' || \ + prefix_name == NULL || *prefix_name == '\0' || \ + group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + return storage_do_upload_file(pTrackerServer, pStorageServer, \ + 0, STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, (char *)callback, arg, \ + file_size, master_filename, prefix_name, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); +} + +int storage_upload_slave_by_filebuff(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename) +{ + if (master_filename == NULL || *master_filename == '\0' || \ + prefix_name == NULL || *prefix_name == '\0' || \ + group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + return storage_do_upload_file(pTrackerServer, pStorageServer, \ + 0, STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_size, master_filename, prefix_name, \ + file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); +} + +int storage_upload_slave_by_filename1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *master_file_id, const char *prefix_name, \ + const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *file_id) +{ + int result; + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(master_file_id) + + strcpy(new_group_name, group_name); + result = storage_upload_slave_by_filename(pTrackerServer, \ + pStorageServer, local_filename, filename, \ + prefix_name, file_ext_name, \ + meta_list, meta_count, \ + new_group_name, remote_filename); + if (result == 0) + { + sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + } + else + { + *file_id = '\0'; + } + + return result; +} + +int storage_upload_slave_by_filebuff1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *master_file_id, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *file_id) +{ + int result; + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(master_file_id) + + strcpy(new_group_name, group_name); + result = storage_upload_slave_by_filebuff(pTrackerServer, \ + pStorageServer, file_buff, file_size, \ + filename, prefix_name, file_ext_name, \ + meta_list, meta_count, \ + new_group_name, remote_filename); + if (result == 0) + { + sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + } + else + { + *file_id = '\0'; + } + + return result; +} + +int storage_upload_slave_by_callback1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_size, const char *master_file_id, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *file_id) +{ + int result; + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(master_file_id) + + strcpy(new_group_name, group_name); + result = storage_upload_slave_by_callback(pTrackerServer, \ + pStorageServer, callback, arg, file_size, \ + filename, prefix_name, file_ext_name, \ + meta_list, meta_count, \ + new_group_name, remote_filename); + if (result == 0) + { + sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + } + else + { + *file_id = '\0'; + } + + return result; +} + +/** +STORAGE_PROTO_CMD_APPEND_FILE: +8 bytes: appender filename length +8 bytes: file size +master filename bytes: appender filename +file size bytes: file content +**/ +int storage_do_append_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int upload_type, \ + const char *file_buff, void *arg, const int64_t file_size, \ + const char *group_name, const char *appender_filename) +{ + TrackerHeader *pHeader; + int result; + char out_buff[512]; + char *p; + int64_t in_bytes; + int64_t total_send_bytes; + ConnectionInfo storageServer; + bool new_connection; + int appender_filename_len; + + appender_filename_len = strlen(appender_filename); + + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, appender_filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + /* + //printf("upload to storage %s:%d\n", \ + pStorageServer->ip_addr, pStorageServer->port); + */ + + do + { + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + long2buff(appender_filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(file_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + memcpy(p, appender_filename, appender_filename_len); + p += appender_filename_len; + + long2buff((p - out_buff) + file_size - sizeof(TrackerHeader), \ + pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_APPEND_FILE; + pHeader->status = 0; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + if ((result=tcpsendfile(pStorageServer->sock, file_buff, \ + file_size, g_fdfs_network_timeout, \ + &total_send_bytes)) != 0) + { + break; + } + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + if ((result=tcpsenddata_nb(pStorageServer->sock, \ + (char *)file_buff, file_size, \ + g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + } + else //FDFS_UPLOAD_BY_CALLBACK + { + UploadCallback callback; + callback = (UploadCallback)file_buff; + if ((result=callback(arg, file_size, pStorageServer->sock))!=0) + { + break; + } + } + + if ((result=fdfs_recv_header(pStorageServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "should == 0", __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, in_bytes); + result = EINVAL; + break; + } + + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +/** +STORAGE_PROTO_CMD_APPEND_FILE: +8 bytes: appender filename length +8 bytes: file offset +8 bytes: file size +master filename bytes: appender filename +file size bytes: file content +**/ +int storage_do_modify_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int upload_type, \ + const char *file_buff, void *arg, const int64_t file_offset, \ + const int64_t file_size, const char *group_name, \ + const char *appender_filename) +{ + TrackerHeader *pHeader; + int result; + char out_buff[512]; + char *p; + int64_t in_bytes; + int64_t total_send_bytes; + ConnectionInfo storageServer; + bool new_connection; + int appender_filename_len; + + appender_filename_len = strlen(appender_filename); + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, appender_filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + /* + //printf("upload to storage %s:%d\n", \ + pStorageServer->ip_addr, pStorageServer->port); + */ + + do + { + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + long2buff(appender_filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(file_offset, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(file_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + memcpy(p, appender_filename, appender_filename_len); + p += appender_filename_len; + + long2buff((p - out_buff) + file_size - sizeof(TrackerHeader), \ + pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_MODIFY_FILE; + pHeader->status = 0; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + if ((result=tcpsendfile(pStorageServer->sock, file_buff, \ + file_size, g_fdfs_network_timeout, \ + &total_send_bytes)) != 0) + { + break; + } + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + if ((result=tcpsenddata_nb(pStorageServer->sock, \ + (char *)file_buff, file_size, \ + g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + } + else //FDFS_UPLOAD_BY_CALLBACK + { + UploadCallback callback; + callback = (UploadCallback)file_buff; + if ((result=callback(arg, file_size, pStorageServer->sock))!=0) + { + break; + } + } + + if ((result=fdfs_recv_header(pStorageServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "should == 0", __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, in_bytes); + result = EINVAL; + break; + } + + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + +int storage_append_by_filename(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *group_name, const char *appender_filename) +{ + struct stat stat_buf; + + if (appender_filename == NULL || *appender_filename == '\0' \ + || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + if (stat(local_filename, &stat_buf) != 0) + { + return errno != 0 ? errno : EPERM; + } + + if (!S_ISREG(stat_buf.st_mode)) + { + return EINVAL; + } + return storage_do_append_file(pTrackerServer, pStorageServer, \ + FDFS_UPLOAD_BY_FILE, local_filename, \ + NULL, stat_buf.st_size, group_name, appender_filename); +} + +int storage_append_by_callback(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, const int64_t file_size, \ + const char *group_name, const char *appender_filename) +{ + if (appender_filename == NULL || *appender_filename == '\0' \ + || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + return storage_do_append_file(pTrackerServer, pStorageServer, \ + FDFS_UPLOAD_BY_CALLBACK, (char *)callback, arg, \ + file_size, group_name, appender_filename); +} + +int storage_append_by_filebuff(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *group_name, \ + const char *appender_filename) +{ + if (appender_filename == NULL || *appender_filename == '\0' \ + || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + return storage_do_append_file(pTrackerServer, pStorageServer, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_size, group_name, appender_filename); +} + +int storage_append_by_filename1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *appender_file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_append_by_filename(pTrackerServer, \ + pStorageServer, local_filename, group_name, filename); +} + +int storage_append_by_filebuff1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *appender_file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_append_by_filebuff(pTrackerServer, \ + pStorageServer, file_buff, file_size, \ + group_name, filename); +} + +int storage_append_by_callback1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_size, const char *appender_file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_append_by_callback(pTrackerServer, \ + pStorageServer, callback, arg, file_size, \ + group_name, filename); +} + +int storage_modify_by_filename(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const int64_t file_offset, const char *group_name, \ + const char *appender_filename) +{ + struct stat stat_buf; + + if (appender_filename == NULL || *appender_filename == '\0' \ + || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + if (stat(local_filename, &stat_buf) != 0) + { + return errno != 0 ? errno : EPERM; + } + + if (!S_ISREG(stat_buf.st_mode)) + { + return EINVAL; + } + return storage_do_modify_file(pTrackerServer, pStorageServer, \ + FDFS_UPLOAD_BY_FILE, local_filename, \ + NULL, file_offset, stat_buf.st_size, \ + group_name, appender_filename); +} + +int storage_modify_by_callback(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, const int64_t file_offset,\ + const int64_t file_size, const char *group_name, \ + const char *appender_filename) +{ + if (appender_filename == NULL || *appender_filename == '\0' \ + || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + return storage_do_modify_file(pTrackerServer, pStorageServer, \ + FDFS_UPLOAD_BY_CALLBACK, (char *)callback, arg, \ + file_offset, file_size, group_name, appender_filename); +} + +int storage_modify_by_filebuff(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_offset, const int64_t file_size, \ + const char *group_name, const char *appender_filename) +{ + if (appender_filename == NULL || *appender_filename == '\0' \ + || group_name == NULL || *group_name == '\0') + { + return EINVAL; + } + + return storage_do_modify_file(pTrackerServer, pStorageServer, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_offset, file_size, group_name, appender_filename); +} + +int storage_modify_by_filename1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const int64_t file_offset, const char *appender_file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_modify_by_filename(pTrackerServer, \ + pStorageServer, local_filename, file_offset, \ + group_name, filename); +} + +int storage_modify_by_filebuff1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_offset, const int64_t file_size, \ + const char *appender_file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_modify_by_filebuff(pTrackerServer, \ + pStorageServer, file_buff, file_offset, file_size, \ + group_name, filename); +} + +int storage_modify_by_callback1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_offset, const int64_t file_size, \ + const char *appender_file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id) + + return storage_modify_by_callback(pTrackerServer, \ + pStorageServer, callback, arg, file_offset, file_size, \ + group_name, filename); +} + +int fdfs_get_file_info_ex1(const char *file_id, const bool get_from_server, \ + FDFSFileInfo *pFileInfo) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + + return fdfs_get_file_info_ex(group_name, filename, get_from_server, \ + pFileInfo); +} + +int fdfs_get_file_info_ex(const char *group_name, const char *remote_filename, \ + const bool get_from_server, FDFSFileInfo *pFileInfo) +{ + struct in_addr ip_addr; + int filename_len; + int buff_len; + int result; + char buff[64]; + + memset(pFileInfo, 0, sizeof(FDFSFileInfo)); + if (!the_base64_context_inited) + { + the_base64_context_inited = 1; + base64_init_ex(&the_base64_context, 0, '-', '_', '.'); + } + + filename_len = strlen(remote_filename); + if (filename_len < FDFS_NORMAL_LOGIC_FILENAME_LENGTH) + { + logError("file: "__FILE__", line: %d, " \ + "filename is too short, length: %d < %d", \ + __LINE__, filename_len, \ + FDFS_NORMAL_LOGIC_FILENAME_LENGTH); + return EINVAL; + } + + memset(buff, 0, sizeof(buff)); + base64_decode_auto(&the_base64_context, (char *)remote_filename + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + buff, &buff_len); + + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = ntohl(buff2int(buff)); + if (fdfs_get_server_id_type(ip_addr.s_addr) == FDFS_ID_TYPE_SERVER_ID) + { + pFileInfo->source_id = ip_addr.s_addr; + if (g_storage_ids_by_id != NULL && g_storage_id_count > 0) + { + char id[16]; + FDFSStorageIdInfo *pStorageId; + + sprintf(id, "%d", pFileInfo->source_id); + pStorageId = fdfs_get_storage_by_id(id); + if (pStorageId != NULL) + { + strcpy(pFileInfo->source_ip_addr, \ + pStorageId->ip_addr); + } + else + { + *(pFileInfo->source_ip_addr) = '\0'; + } + } + else + { + *(pFileInfo->source_ip_addr) = '\0'; + } + } + else + { + pFileInfo->source_id = 0; + inet_ntop(AF_INET, &ip_addr, pFileInfo->source_ip_addr, \ + IP_ADDRESS_SIZE); + } + + pFileInfo->create_timestamp = buff2int(buff + sizeof(int)); + pFileInfo->file_size = buff2long(buff + sizeof(int) * 2); + + if (IS_SLAVE_FILE(filename_len, pFileInfo->file_size) || \ + IS_APPENDER_FILE(pFileInfo->file_size) || \ + (*(pFileInfo->source_ip_addr) == '\0' && get_from_server)) + { //slave file or appender file + if (get_from_server) + { + ConnectionInfo *conn; + ConnectionInfo trackerServer; + + conn = tracker_get_connection_r(&trackerServer, &result); + if (result != 0) + { + return result; + } + + result = storage_query_file_info(conn, \ + NULL, group_name, remote_filename, pFileInfo); + tracker_disconnect_server_ex(conn, result != 0 && \ + result != ENOENT); + + return result; + } + else + { + pFileInfo->file_size = -1; + return 0; + } + } + else //master file (normal file) + { + if ((pFileInfo->file_size >> 63) != 0) + { + pFileInfo->file_size &= 0xFFFFFFFF; //low 32 bits is file size + } + else if (IS_TRUNK_FILE(pFileInfo->file_size)) + { + pFileInfo->file_size = FDFS_TRUNK_FILE_TRUE_SIZE( \ + pFileInfo->file_size); + } + + pFileInfo->crc32 = buff2int(buff+sizeof(int)*4); + } + + return 0; +} + +int storage_file_exist(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *remote_filename) +{ + FDFSFileInfo file_info; + return storage_query_file_info_ex(pTrackerServer, \ + pStorageServer, group_name, remote_filename, \ + &file_info, true); +} + +int storage_file_exist1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id) +{ + FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) + return storage_file_exist(pTrackerServer, pStorageServer, \ + group_name, filename); +} + +int storage_truncate_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, + const char *group_name, const char *appender_filename, \ + const int64_t truncated_file_size) +{ + TrackerHeader *pHeader; + int result; + char out_buff[512]; + char *p; + int64_t in_bytes; + ConnectionInfo storageServer; + bool new_connection; + int appender_filename_len; + + appender_filename_len = strlen(appender_filename); + if ((result=storage_get_update_connection(pTrackerServer, \ + &pStorageServer, group_name, appender_filename, \ + &storageServer, &new_connection)) != 0) + { + return result; + } + + /* + //printf("upload to storage %s:%d\n", \ + pStorageServer->ip_addr, pStorageServer->port); + */ + + do + { + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + long2buff(appender_filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(truncated_file_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + memcpy(p, appender_filename, appender_filename_len); + p += appender_filename_len; + + long2buff((p - out_buff) - sizeof(TrackerHeader), \ + pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_TRUNCATE_FILE; + pHeader->status = 0; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pStorageServer->ip_addr, pStorageServer->port, \ + result, STRERROR(result)); + break; + } + + if ((result=fdfs_recv_header(pStorageServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "should == 0", __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, in_bytes); + result = EINVAL; + break; + } + } while (0); + + if (new_connection) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + + return result; +} + diff --git a/client/storage_client.h b/client/storage_client.h new file mode 100644 index 0000000..12489a9 --- /dev/null +++ b/client/storage_client.h @@ -0,0 +1,571 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef STORAGE_CLIENT_H +#define STORAGE_CLIENT_H + +#include "tracker_types.h" +#include "client_func.h" + +#define FDFS_DOWNLOAD_TO_BUFF 1 +#define FDFS_DOWNLOAD_TO_FILE 2 +#define FDFS_DOWNLOAD_TO_CALLBACK 3 + +#define FDFS_UPLOAD_BY_BUFF 1 +#define FDFS_UPLOAD_BY_FILE 2 +#define FDFS_UPLOAD_BY_CALLBACK 3 + +#define FDFS_FILE_ID_SEPERATOR '/' +#define FDFS_FILE_ID_SEPERATE_STR "/" + +#ifdef __cplusplus +extern "C" { +#endif + +#define storage_upload_by_filename(pTrackerServer, \ + pStorageServer, store_path_index, local_filename, \ + file_ext_name, meta_list, meta_count, group_name, \ + remote_filename) \ + storage_upload_by_filename_ex(pTrackerServer, \ + pStorageServer, store_path_index, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, local_filename, \ + file_ext_name, meta_list, meta_count, group_name, \ + remote_filename) + +#define storage_upload_appender_by_filename(pTrackerServer, \ + pStorageServer, store_path_index, local_filename, \ + file_ext_name, meta_list, meta_count, group_name, \ + remote_filename) \ + storage_upload_by_filename_ex(pTrackerServer, \ + pStorageServer, store_path_index, \ + STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, local_filename, \ + file_ext_name, meta_list, meta_count, group_name, \ + remote_filename) + +/** +* upload file to storage server (by file name) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* store_path_index: the index of path on the storage server +* local_filename: local filename to upload +* cmd: the protocol command +* file_ext_name: file ext name, not include dot(.), +* if be NULL will abstract ext name from the local filename +* meta_list: meta info array +* meta_count: meta item count +* group_name: if not empty, specify the group name. + return the group name to store the file +* remote_filename: return the new created filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_by_filename_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const char *local_filename, \ + const char *file_ext_name, const FDFSMetaData *meta_list, \ + const int meta_count, char *group_name, char *remote_filename); + +/** +* upload file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* store_path_index: the index of path on the storage server +* file_buff: file content/buff +* file_size: file size (bytes) +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* group_name: if not empty, specify the group name. + return the group name to store the file +* remote_filename: return the new created filename +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_upload_by_filebuff(pTrackerServer, pStorageServer, \ + store_path_index, file_buff, \ + file_size, file_ext_name, meta_list, meta_count, \ + group_name, remote_filename) \ + storage_do_upload_file(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_size, NULL, NULL, file_ext_name, meta_list, meta_count, \ + group_name, remote_filename) + +#define storage_upload_appender_by_filebuff(pTrackerServer, pStorageServer, \ + store_path_index, file_buff, \ + file_size, file_ext_name, meta_list, meta_count, \ + group_name, remote_filename) \ + storage_do_upload_file(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_size, NULL, NULL, file_ext_name, meta_list, meta_count, \ + group_name, remote_filename) + +/** +* Upload file callback function prototype +* params: +* arg: callback extra arguement +* sock: connected storage socket for sending file content +* return: 0 success, !=0 fail, should return the error code +**/ +typedef int (*UploadCallback) (void *arg, const int64_t file_size, int sock); + +/** +* upload file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* store_path_index: the index of path on the storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_size: the file size +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* group_name: if not empty, specify the group name. + return the group name to store the file +* remote_filename: return the new created filename +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_upload_by_callback(pTrackerServer, pStorageServer, \ + store_path_index, callback, arg, file_size, file_ext_name, \ + meta_list, meta_count, group_name, remote_filename) \ + storage_upload_by_callback_ex(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + callback, arg, file_size, file_ext_name, meta_list, \ + meta_count, group_name, remote_filename) + +#define storage_upload_appender_by_callback(pTrackerServer, pStorageServer, \ + store_path_index, callback, arg, file_size, file_ext_name, \ + meta_list, meta_count, group_name, remote_filename) \ + storage_upload_by_callback_ex(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + callback, arg, file_size, file_ext_name, meta_list, \ + meta_count, group_name, remote_filename) + +int storage_upload_by_callback_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, UploadCallback callback, void *arg, \ + const int64_t file_size, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename); + +int storage_do_upload_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const int upload_type, const char *file_buff, \ + void *arg, const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename); + +/** +* delete file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* filename: filename on storage server +* return: 0 success, !=0 fail, return the error code +**/ +int storage_delete_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename); + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* remote_filename: filename on storage server +* file_buff: return file content/buff, must be freed +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_download_file(pTrackerServer, pStorageServer, group_name, \ + remote_filename, file_buff, file_size) \ + storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_BUFF, group_name, remote_filename, \ + 0, 0, file_buff, NULL, file_size) + +#define storage_download_file_to_buff(pTrackerServer, pStorageServer, \ + group_name, remote_filename, file_buff, file_size) \ + storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_BUFF, group_name, remote_filename, \ + 0, 0, file_buff, NULL, file_size) + +#define storage_do_download_file(pTrackerServer, pStorageServer, \ + download_type, group_name, remote_filename, \ + file_buff, arg, file_size) \ + storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + download_type, group_name, remote_filename, \ + 0, 0, file_buff, arg, file_size); + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* download_type: FDFS_DOWNLOAD_TO_BUFF or FDFS_DOWNLOAD_TO_FILE +* or FDFS_DOWNLOAD_TO_CALLBACK +* group_name: the group name of storage server +* remote_filename: filename on storage server +* file_offset: the start offset to download +* download_bytes: download bytes, 0 means from start offset to the file end +* file_buff: return file content/buff, must be freed +* arg: additional argument for callback(valid only when download_tyee +* is FDFS_DOWNLOAD_TO_CALLBACK), can be NULL +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_do_download_file_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const int download_type, \ + const char *group_name, const char *remote_filename, \ + const int64_t file_offset, const int64_t download_bytes, \ + char **file_buff, void *arg, int64_t *file_size); + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* remote_filename: filename on storage server +* local_filename: local filename to write +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_download_file_to_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *remote_filename, \ + const char *local_filename, int64_t *file_size); + +/** +* Download file callback function prototype +* params: +* arg: callback extra arguement +* file_size: file size +* data: temp buff, should not keep persistently +* current_size: current data size +* return: 0 success, !=0 fail, should return the error code +**/ +typedef int (*DownloadCallback) (void *arg, const int64_t file_size, \ + const char *data, const int current_size); + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* remote_filename: filename on storage server +* file_offset: the start offset to download +* download_bytes: download bytes, 0 means from start offset to the file end +* callback: callback function +* arg: callback extra arguement +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_download_file_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *remote_filename, \ + const int64_t file_offset, const int64_t download_bytes, \ + DownloadCallback callback, void *arg, int64_t *file_size); + +/** +* set metadata items to storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* filename: filename on storage server +* meta_list: meta item array +* meta_count: meta item count +* op_flag: +* # STORAGE_SET_METADATA_FLAG_OVERWRITE('O'): overwrite all old +* metadata items +* # STORAGE_SET_METADATA_FLAG_MERGE ('M'): merge, insert when +* the metadata item not exist, otherwise update it +* return: 0 success, !=0 fail, return the error code +**/ +int storage_set_metadata(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename, \ + const FDFSMetaData *meta_list, const int meta_count, \ + const char op_flag); + +/** +* get all metadata items from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* filename: filename on storage server +* meta_list: return meta info array, must be freed +* meta_count: return meta item count +* return: 0 success, !=0 fail, return the error code +**/ +int storage_get_metadata(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename, \ + FDFSMetaData **meta_list, \ + int *meta_count); + +/** +* upload slave file to storage server (by file name) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* local_filename: local filename to upload +* master_filename: the mater filename to generate the slave file id +* prefix_name: the prefix name to generate the slave file id +* file_ext_name: file ext name, not include dot(.), +* if be NULL will abstract ext name from the local filename +* meta_list: meta info array +* meta_count: meta item count +* group_name: specify the group name. + return the group name to store the file +* remote_filename: return the new created filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_slave_by_filename(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *master_filename, const char *prefix_name, \ + const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename); + +/** +* upload slave file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_buff: file content/buff +* file_size: file size (bytes) +* master_filename: the mater filename to generate the slave file id +* prefix_name: the prefix name to generate the slave file id +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* group_name: specify the group name. + return the group name to store the file +* remote_filename: return the new created filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_slave_by_filebuff(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename); + +/** +* upload slave file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_size: the file size +* master_filename: the mater filename to generate the slave file id +* prefix_name: the prefix name to generate the slave file id +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* group_name: specify the group name. + return the group name to store the file +* remote_filename: return the new created filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_slave_by_callback(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *group_name, char *remote_filename); + + +/** +* append file to storage server (by local filename) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* local_filename: local filename to upload +* group_name: the group name +* appender_filename: the appender filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_append_by_filename(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *group_name, const char *appender_filename); + + +/** +* append file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_size: the file size +* group_name: the group name +* appender_filename: the appender filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_append_by_callback(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, const int64_t file_size, \ + const char *group_name, const char *appender_filename); + + +/** +* append file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_buff: file content/buff +* file_size: file size (bytes) +* group_name: the group name +* appender_filename: the appender filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_append_by_filebuff(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *group_name, \ + const char *appender_filename); + + +/** +* modify file to storage server (by local filename) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* local_filename: local filename to upload +* file_offset: the start offset to modify appender file +* group_name: the group name +* appender_filename: the appender filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_modify_by_filename(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const int64_t file_offset, const char *group_name, \ + const char *appender_filename); + + +/** +* modify file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_offset: the start offset to modify appender file +* file_size: the file size +* group_name: the group name +* appender_filename: the appender filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_modify_by_callback(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_offset, const int64_t file_size, \ + const char *group_name, const char *appender_filename); + + +/** +* modify file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_buff: file content/buff +* file_offset: the start offset to modify appender file +* file_size: file size (bytes) +* group_name: the group name +* appender_filename: the appender filename +* return: 0 success, !=0 fail, return the error code +**/ +int storage_modify_by_filebuff(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_offset, const int64_t file_size, \ + const char *group_name, const char *appender_filename); + + +/** +* truncate file to sepecify size +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name +* appender_filename: the appender filename +* truncated_file_size: truncated file size +* return: 0 success, !=0 fail, return the error code +**/ +int storage_truncate_file(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, + const char *group_name, const char *appender_filename, \ + const int64_t truncated_file_size); + + +#define storage_query_file_info(pTrackerServer, pStorageServer, \ + group_name, filename, pFileInfo) \ + storage_query_file_info_ex(pTrackerServer, pStorageServer, \ + group_name, filename, pFileInfo, false) + +/** +* query file info +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* filename: filename on storage server +* pFileInfo: return the file info (file size and create timestamp) +* bSilence: when this file not exist, do not log error on storage server +* return: 0 success, !=0 fail, return the error code +**/ +int storage_query_file_info_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *filename, \ + FDFSFileInfo *pFileInfo, const bool bSilence); + + +#define fdfs_get_file_info(group_name, remote_filename, pFileInfo) \ + fdfs_get_file_info_ex(group_name, remote_filename, true, pFileInfo) + +/** +* check if file exist +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* group_name: the group name of storage server +* remote_filename: filename on storage server +* return: 0 file exist, !=0 not exist, return the error code +**/ +int storage_file_exist(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *group_name, const char *remote_filename); +/** +* get file info from the filename return by storage server +* params: +* group_name: the group name of storage server +* remote_filename: filename on storage server +* get_from_server: if get slave file info from storage server +* pFileInfo: return the file info +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_get_file_info_ex(const char *group_name, const char *remote_filename, \ + const bool get_from_server, FDFSFileInfo *pFileInfo); + + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/client/storage_client1.h b/client/storage_client1.h new file mode 100644 index 0000000..63b5e4c --- /dev/null +++ b/client/storage_client1.h @@ -0,0 +1,532 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef STORAGE_CLIENT1_H +#define STORAGE_CLIENT1_H + +#include "tracker_types.h" +#include "storage_client.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** +* upload file to storage server (by file name) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* store_path_index: the index of path on the storage server +* local_filename: local filename to upload +* file_ext_name: file ext name, not include dot(.), +* if be NULL will abstract ext name from the local filename +* meta_list: meta info array +* meta_count: meta item count +* group_name: specify the group name to upload file to, can be NULL or emtpy +* file_id: return the new created file id (including group name and filename) +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_upload_by_filename1(pTrackerServer, pStorageServer, \ + store_path_index, local_filename, file_ext_name, \ + meta_list, meta_count, group_name, file_id) \ + storage_upload_by_filename1_ex(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + local_filename, file_ext_name, meta_list, meta_count, \ + group_name, file_id) + +#define storage_upload_appender_by_filename1(pTrackerServer, pStorageServer, \ + store_path_index, local_filename, file_ext_name, \ + meta_list, meta_count, group_name, file_id) \ + storage_upload_by_filename1_ex(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + local_filename, file_ext_name, meta_list, meta_count, \ + group_name, file_id) + +int storage_upload_by_filename1_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const char *local_filename, \ + const char *file_ext_name, const FDFSMetaData *meta_list, \ + const int meta_count, const char *group_name, char *file_id); + +/** +* upload file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* store_path_index: the index of path on the storage server +* file_buff: file content/buff +* file_size: file size (bytes) +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* group_name: specify the group name to upload file to, can be NULL or emtpy +* file_id: return the new created file id (including group name and filename) +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_upload_by_filebuff1(pTrackerServer, pStorageServer, \ + store_path_index, file_buff, file_size, file_ext_name, \ + meta_list, meta_count, group_name, file_id) \ + storage_do_upload_file1(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_size, file_ext_name, meta_list, meta_count, \ + group_name, file_id) + +#define storage_upload_appender_by_filebuff1(pTrackerServer, pStorageServer, \ + store_path_index, file_buff, file_size, file_ext_name, \ + meta_list, meta_count, group_name, file_id) \ + storage_do_upload_file1(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_BUFF, file_buff, NULL, \ + file_size, file_ext_name, meta_list, meta_count, \ + group_name, file_id) + +int storage_do_upload_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, const int upload_type, \ + const char *file_buff, void *arg, const int64_t file_size, \ + const char *file_ext_name, const FDFSMetaData *meta_list, \ + const int meta_count, const char *group_name, char *file_id); + +/** +* upload file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* store_path_index: the index of path on the storage server +* file_size: the file size +* file_ext_name: file ext name, not include dot(.), can be NULL +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* meta_list: meta info array +* meta_count: meta item count +* group_name: specify the group name to upload file to, can be NULL or emtpy +* file_id: return the new created file id (including group name and filename) +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_upload_by_callback1(pTrackerServer, pStorageServer, \ + store_path_index, callback, arg, \ + file_size, file_ext_name, meta_list, meta_count, \ + group_name, file_id) \ + storage_upload_by_callback1_ex(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + callback, arg, file_size, file_ext_name, meta_list, \ + meta_count, group_name, file_id) + +#define storage_upload_appender_by_callback1(pTrackerServer, pStorageServer, \ + store_path_index, callback, arg, \ + file_size, file_ext_name, meta_list, meta_count, \ + group_name, file_id) \ + storage_upload_by_callback1_ex(pTrackerServer, pStorageServer, \ + store_path_index, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + callback, arg, file_size, file_ext_name, meta_list, \ + meta_count, group_name, file_id) + +int storage_upload_by_callback1_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int store_path_index, \ + const char cmd, UploadCallback callback, void *arg, \ + const int64_t file_size, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + const char *group_name, char *file_id); + +/** +* delete file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id to deleted (including group name and filename) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_delete_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id); + +/** +* delete file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* appender_file_id: the appender file id +* truncated_file_size: the truncated file size +* return: 0 success, !=0 fail, return the error code +**/ +int storage_truncate_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, + const char *appender_file_id, \ + const int64_t truncated_file_size); + +/** +* set metadata items to storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id (including group name and filename) +* meta_list: meta item array +* meta_count: meta item count +* op_flag: +* # STORAGE_SET_METADATA_FLAG_OVERWRITE('O'): overwrite all old +* metadata items +* # STORAGE_SET_METADATA_FLAG_MERGE ('M'): merge, insert when +* the metadata item not exist, otherwise update it +* return: 0 success, !=0 fail, return the error code +**/ +int storage_set_metadata1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + const FDFSMetaData *meta_list, const int meta_count, \ + const char op_flag); + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id (including group name and filename) +* file_buff: return file content/buff, must be freed +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +#define storage_download_file1(pTrackerServer, pStorageServer, file_id, \ + file_buff, file_size) \ + storage_do_download_file1_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_BUFF, file_id, 0, 0, \ + file_buff, NULL, file_size) + +#define storage_download_file_to_buff1(pTrackerServer, pStorageServer, \ + file_id, file_buff, file_size) \ + storage_do_download_file1_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_BUFF, file_id, 0, 0, \ + file_buff, NULL, file_size) + +#define storage_do_download_file1(pTrackerServer, pStorageServer, \ + download_type, file_id, file_buff, file_size) \ + storage_do_download_file1_ex(pTrackerServer, pStorageServer, \ + download_type, file_id, \ + 0, 0, file_buff, NULL, file_size) + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id (including group name and filename) +* file_offset: the start offset to download +* download_bytes: download bytes, 0 means from start offset to the file end +* file_buff: return file content/buff, must be freed +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_do_download_file1_ex(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const int download_type, const char *file_id, \ + const int64_t file_offset, const int64_t download_bytes, \ + char **file_buff, void *arg, int64_t *file_size); + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id (including group name and filename) +* local_filename: local filename to write +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_download_file_to_file1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + const char *local_filename, int64_t *file_size); + +/** +* get all metadata items from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id (including group name and filename) +* meta_list: return meta info array, must be freed +* meta_count: return meta item count +* return: 0 success, !=0 fail, return the error code +**/ +int storage_get_metadata1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + FDFSMetaData **meta_list, int *meta_count); + + +/** +* download file from storage server +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id (including group name and filename) +* file_offset: the start offset to download +* download_bytes: download bytes, 0 means from start offset to the file end +* callback: callback function +* arg: callback extra arguement +* file_size: return file size (bytes) +* return: 0 success, !=0 fail, return the error code +**/ +int storage_download_file_ex1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id, \ + const int64_t file_offset, const int64_t download_bytes, \ + DownloadCallback callback, void *arg, int64_t *file_size); + +/** +* query storage server to download file +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* file_id: the file id (including group name and filename) +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_fetch1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id); + +/** +* query storage server to update (delete file and set metadata) +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* file_id: the file id (including group name and filename) +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_update1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id); + +/** +* query storage server list to fetch file +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* nMaxServerCount: max storage server count +* server_count: return storage server count +* group_name: the group name of storage server +* filename: filename on storage server +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_list1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int nMaxServerCount, \ + int *server_count, const char *file_id); + +/** +* upload slave file to storage server (by file name) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* local_filename: local filename to upload +* master_file_id: the mater file id to generate the slave file id +* prefix_name: the prefix name to generate the file id +* file_ext_name: file ext name, not include dot(.), +* if be NULL will abstract ext name from the local filename +* meta_list: meta info array +* meta_count: meta item count +* file_id: return the slave file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_slave_by_filename1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *master_file_id, const char *prefix_name, \ + const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *file_id); + +/** +* upload slave file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_buff: file content/buff +* file_size: file size (bytes) +* master_file_id: the mater file id to generate the slave file id +* prefix_name: the prefix name to generate the file id +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* file_id: return the slave file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_slave_by_filebuff1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *master_file_id, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *file_id); + +/** +* upload slave file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_size: the file size +* master_file_id: the mater file id to generate the slave file id +* prefix_name: the prefix name to generate the file id +* file_ext_name: file ext name, not include dot(.), can be NULL +* meta_list: meta info array +* meta_count: meta item count +* file_id: return the slave file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_upload_slave_by_callback1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_size, const char *master_file_id, \ + const char *prefix_name, const char *file_ext_name, \ + const FDFSMetaData *meta_list, const int meta_count, \ + char *file_id); + +/** +* append file to storage server (by filename) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* local_filename: local filename to upload +* appender_file_id: the appender file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_append_by_filename1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const char *appender_file_id); + + +/** +* append file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_buff: file content/buff +* file_size: file size (bytes) +* appender_file_id: the appender file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_append_by_filebuff1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_size, const char *appender_file_id); + + +/** +* append file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_size: the file size +* appender_file_id: the appender file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_append_by_callback1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_size, const char *appender_file_id); + +/** +* modify file to storage server (by local filename) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* local_filename: local filename to upload +* file_offset: the start offset to modify appender file +* appender_file_id: the appender file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_modify_by_filename1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *local_filename,\ + const int64_t file_offset, const char *appender_file_id); + + +/** +* modify file to storage server (by callback) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* callback: callback function to send file content to storage server +* arg: callback extra arguement +* file_offset: the start offset to modify appender file +* file_size: the file size +* appender_file_id: the appender file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_modify_by_callback1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + UploadCallback callback, void *arg, \ + const int64_t file_offset, const int64_t file_size, \ + const char *appender_file_id); + + +/** +* modify file to storage server (by file buff) +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_buff: file content/buff +* file_offset: the start offset to modify appender file +* file_size: file size (bytes) +* appender_file_id: the appender file id +* return: 0 success, !=0 fail, return the error code +**/ +int storage_modify_by_filebuff1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_buff, \ + const int64_t file_offset, const int64_t file_size, \ + const char *appender_file_id); + + +#define storage_query_file_info1(pTrackerServer, pStorageServer, file_id, \ + pFileInfo) \ + storage_query_file_info_ex1(pTrackerServer, pStorageServer, file_id, \ + pFileInfo, false) + +/** +* query file info +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id +* pFileInfo: return the file info (file size and create timestamp) +* bSilence: when this file not exist, do not log error on storage server +* return: 0 success, !=0 fail, return the error code +**/ +int storage_query_file_info_ex1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *file_id, \ + FDFSFileInfo *pFileInfo, const bool bSilence); + +#define fdfs_get_file_info1(file_id, pFileInfo) \ + fdfs_get_file_info_ex1(file_id, true, pFileInfo) + +/** +* get file info from the filename return by storage server +* params: +* file_id: the file id return by storage server +* get_from_server: if get slave file info from storage server +* pFileInfo: return the file info +* return: 0 success, !=0 fail, return the error code +**/ +int fdfs_get_file_info_ex1(const char *file_id, const bool get_from_server, \ + FDFSFileInfo *pFileInfo); + +/** +* check if file exist +* params: +* pTrackerServer: tracker server +* pStorageServer: storage server +* file_id: the file id return by storage server +* return: 0 file exist, !=0 not exist, return the error code +**/ +int storage_file_exist1(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, \ + const char *file_id); +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/client/test/Makefile b/client/test/Makefile new file mode 100644 index 0000000..423d643 --- /dev/null +++ b/client/test/Makefile @@ -0,0 +1,23 @@ +.SUFFIXES: .c .o + +COMPILE = $(CC) -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -g -O -DDEBUG_FLAG -DOS_LINUX -DIOEVENT_USE_EPOLL +INC_PATH = -I/usr/local/include/fastcommon -I/usr/local/include/fastdfs +LIB_PATH = -L/usr/local/lib -lfastcommon -lfdfsclient -lpthread -ldl -rdynamic +TARGET_PATH = $(TARGET_PATH) + +ALL_OBJS = + +ALL_PRGS = fdfs_monitor fdfs_test fdfs_test1 + +all: $(ALL_OBJS) $(ALL_PRGS) +.o: + $(COMPILE) -o $@ $< $(SHARED_OBJS) $(LIB_PATH) $(INC_PATH) +.c: + $(COMPILE) -o $@ $< $(ALL_OBJS) $(LIB_PATH) $(INC_PATH) +.c.o: + $(COMPILE) -c -o $@ $< $(INC_PATH) +install: + cp -f $(ALL_PRGS) $(TARGET_PATH) +clean: + rm -f $(ALL_OBJS) $(ALL_PRGS) + diff --git a/client/test/Makefile.in b/client/test/Makefile.in new file mode 100644 index 0000000..831e7b0 --- /dev/null +++ b/client/test/Makefile.in @@ -0,0 +1,23 @@ +.SUFFIXES: .c .o + +COMPILE = $(CC) $(CFLAGS) +INC_PATH = -I/usr/local/include/fastcommon -I/usr/local/include/fastdfs +LIB_PATH = -L/usr/local/lib -lfastcommon -lfdfsclient $(LIBS) +TARGET_PATH = $(TARGET_PATH) + +ALL_OBJS = + +ALL_PRGS = fdfs_monitor fdfs_test fdfs_test1 + +all: $(ALL_OBJS) $(ALL_PRGS) +.o: + $(COMPILE) -o $@ $< $(SHARED_OBJS) $(LIB_PATH) $(INC_PATH) +.c: + $(COMPILE) -o $@ $< $(ALL_OBJS) $(LIB_PATH) $(INC_PATH) +.c.o: + $(COMPILE) -c -o $@ $< $(INC_PATH) +install: + cp -f $(ALL_PRGS) $(TARGET_PATH) +clean: + rm -f $(ALL_OBJS) $(ALL_PRGS) + diff --git a/client/test/fdfs_monitor.c b/client/test/fdfs_monitor.c new file mode 120000 index 0000000..9f42ff4 --- /dev/null +++ b/client/test/fdfs_monitor.c @@ -0,0 +1 @@ +../fdfs_monitor.c \ No newline at end of file diff --git a/client/test/fdfs_test.c b/client/test/fdfs_test.c new file mode 120000 index 0000000..c3a3f16 --- /dev/null +++ b/client/test/fdfs_test.c @@ -0,0 +1 @@ +../fdfs_test.c \ No newline at end of file diff --git a/client/test/fdfs_test1.c b/client/test/fdfs_test1.c new file mode 120000 index 0000000..c87e07d --- /dev/null +++ b/client/test/fdfs_test1.c @@ -0,0 +1 @@ +../fdfs_test1.c \ No newline at end of file diff --git a/client/tracker_client.c b/client/tracker_client.c new file mode 100644 index 0000000..55449fc --- /dev/null +++ b/client/tracker_client.c @@ -0,0 +1,1529 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "shared_func.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_client.h" +#include "client_global.h" + +int tracker_get_all_connections_ex(TrackerServerGroup *pTrackerGroup) +{ + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + int success_count; + + success_count = 0; + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pTrackerGroup->servers; pServersock >= 0) + { + success_count++; + } + else if (conn_pool_connect_server(pServer, \ + g_fdfs_connect_timeout) == 0) + { + fdfs_active_test(pServer); + success_count++; + } + } + + return success_count > 0 ? 0 : ENOTCONN; +} + +void tracker_close_all_connections_ex(TrackerServerGroup *pTrackerGroup) +{ + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pTrackerGroup->servers; pServerserver_index; + if (server_index >= pTrackerGroup->server_count) + { + server_index = 0; + } + + do + { + pCurrentServer = pTrackerGroup->servers + server_index; + if ((conn=tracker_connect_server(pCurrentServer, &result)) != NULL) + { + break; + } + + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pCurrentServer+1; pServerserver_index = pServer - \ + pTrackerGroup->servers; + break; + } + } + + if (conn != NULL) + { + break; + } + + for (pServer=pTrackerGroup->servers; pServerserver_index = pServer - \ + pTrackerGroup->servers; + break; + } + } + } while (0); + + pTrackerGroup->server_index++; + if (pTrackerGroup->server_index >= pTrackerGroup->server_count) + { + pTrackerGroup->server_index = 0; + } + + return conn; +} + +ConnectionInfo *tracker_get_connection_no_pool(TrackerServerGroup *pTrackerGroup) +{ + ConnectionInfo *pCurrentServer; + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + ConnectionInfo *conn; + int server_index; + int result; + + server_index = pTrackerGroup->server_index; + if (server_index >= pTrackerGroup->server_count) + { + server_index = 0; + } + + conn = NULL; + do + { + pCurrentServer = pTrackerGroup->servers + server_index; + if ((result=tracker_connect_server_no_pool(pCurrentServer)) == 0) + { + conn = pCurrentServer; + break; + } + + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pCurrentServer+1; pServerserver_index = pServer - \ + pTrackerGroup->servers; + break; + } + } + + if (conn != NULL) + { + break; + } + + for (pServer=pTrackerGroup->servers; pServerserver_index = pServer - \ + pTrackerGroup->servers; + break; + } + } + } while (0); + + pTrackerGroup->server_index++; + if (pTrackerGroup->server_index >= pTrackerGroup->server_count) + { + pTrackerGroup->server_index = 0; + } + + return conn; +} + +ConnectionInfo *tracker_get_connection_r_ex(TrackerServerGroup *pTrackerGroup, \ + ConnectionInfo *pTrackerServer, int *err_no) +{ + ConnectionInfo *conn; + ConnectionInfo *pCurrentServer; + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + int server_index; + + server_index = pTrackerGroup->server_index; + if (server_index >= pTrackerGroup->server_count) + { + server_index = 0; + } + + do + { + pCurrentServer = pTrackerGroup->servers + server_index; + memcpy(pTrackerServer, pCurrentServer, sizeof(ConnectionInfo)); + pTrackerServer->sock = -1; + if ((conn=tracker_connect_server(pTrackerServer, err_no)) != NULL) + { + break; + } + + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pCurrentServer+1; pServersock = -1; + if ((conn=tracker_connect_server(pTrackerServer, err_no)) != NULL) + { + pTrackerGroup->server_index = pServer - \ + pTrackerGroup->servers; + break; + } + } + + if (conn != NULL) + { + break; + } + + for (pServer=pTrackerGroup->servers; pServersock = -1; + if ((conn=tracker_connect_server(pTrackerServer, err_no)) != NULL) + { + pTrackerGroup->server_index = pServer - \ + pTrackerGroup->servers; + break; + } + } + } while (0); + + pTrackerGroup->server_index++; + if (pTrackerGroup->server_index >= pTrackerGroup->server_count) + { + pTrackerGroup->server_index = 0; + } + + return conn; +} + +int tracker_list_servers(ConnectionInfo *pTrackerServer, \ + const char *szGroupName, const char *szStorageId, \ + FDFSStorageInfo *storage_infos, const int max_storages, \ + int *storage_count) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + IP_ADDRESS_SIZE]; + bool new_connection; + TrackerHeader *pHeader; + ConnectionInfo *conn; + int result; + int name_len; + int id_len; + TrackerStorageStat stats[FDFS_MAX_GROUPS]; + char *pInBuff; + TrackerStorageStat *pSrc; + TrackerStorageStat *pEnd; + FDFSStorageStat *pStorageStat; + FDFSStorageInfo *pDest; + FDFSStorageStatBuff *pStatBuff; + int64_t in_bytes; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + name_len = strlen(szGroupName); + if (name_len > FDFS_GROUP_NAME_MAX_LEN) + { + name_len = FDFS_GROUP_NAME_MAX_LEN; + } + memcpy(out_buff + sizeof(TrackerHeader), szGroupName, name_len); + + if (szStorageId == NULL) + { + id_len = 0; + } + else + { + id_len = strlen(szStorageId); + if (id_len >= FDFS_STORAGE_ID_MAX_SIZE) + { + id_len = FDFS_STORAGE_ID_MAX_SIZE - 1; + } + + memcpy(out_buff+sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN,\ + szStorageId, id_len); + } + + long2buff(FDFS_GROUP_NAME_MAX_LEN + id_len, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_SERVER_LIST_STORAGE; + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + id_len, \ + g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = (char *)stats; + result = fdfs_recv_response(conn, &pInBuff, \ + sizeof(stats), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + *storage_count = 0; + return result; + } + + if (in_bytes % sizeof(TrackerStorageStat) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + *storage_count = 0; + return EINVAL; + } + + *storage_count = in_bytes / sizeof(TrackerStorageStat); + if (*storage_count > max_storages) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d insufficent space, " \ + "max storage count: %d, expect count: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, max_storages, *storage_count); + *storage_count = 0; + return ENOSPC; + } + + memset(storage_infos, 0, sizeof(FDFSStorageInfo) * max_storages); + pDest = storage_infos; + pEnd = stats + (*storage_count); + for (pSrc=stats; pSrcstat_buff); + pStorageStat = &(pDest->stat); + + pDest->status = pSrc->status; + memcpy(pDest->id, pSrc->id, FDFS_STORAGE_ID_MAX_SIZE - 1); + memcpy(pDest->ip_addr, pSrc->ip_addr, IP_ADDRESS_SIZE - 1); + memcpy(pDest->src_id, pSrc->src_id, \ + FDFS_STORAGE_ID_MAX_SIZE - 1); + strcpy(pDest->domain_name, pSrc->domain_name); + strcpy(pDest->version, pSrc->version); + pDest->join_time = buff2long(pSrc->sz_join_time); + pDest->up_time = buff2long(pSrc->sz_up_time); + pDest->total_mb = buff2long(pSrc->sz_total_mb); + pDest->free_mb = buff2long(pSrc->sz_free_mb); + pDest->upload_priority = buff2long(pSrc->sz_upload_priority); + pDest->store_path_count = buff2long(pSrc->sz_store_path_count); + pDest->subdir_count_per_path = buff2long( \ + pSrc->sz_subdir_count_per_path); + pDest->storage_port = buff2long(pSrc->sz_storage_port); + pDest->storage_http_port = buff2long(pSrc->sz_storage_http_port); + pDest->current_write_path = buff2long( \ + pSrc->sz_current_write_path); + + pStorageStat->total_upload_count = buff2long( \ + pStatBuff->sz_total_upload_count); + pStorageStat->success_upload_count = buff2long( \ + pStatBuff->sz_success_upload_count); + pStorageStat->total_append_count = buff2long( \ + pStatBuff->sz_total_append_count); + pStorageStat->success_append_count = buff2long( \ + pStatBuff->sz_success_append_count); + pStorageStat->total_modify_count = buff2long( \ + pStatBuff->sz_total_modify_count); + pStorageStat->success_modify_count = buff2long( \ + pStatBuff->sz_success_modify_count); + pStorageStat->total_truncate_count = buff2long( \ + pStatBuff->sz_total_truncate_count); + pStorageStat->success_truncate_count = buff2long( \ + pStatBuff->sz_success_truncate_count); + pStorageStat->total_set_meta_count = buff2long( \ + pStatBuff->sz_total_set_meta_count); + pStorageStat->success_set_meta_count = buff2long( \ + pStatBuff->sz_success_set_meta_count); + pStorageStat->total_delete_count = buff2long( \ + pStatBuff->sz_total_delete_count); + pStorageStat->success_delete_count = buff2long( \ + pStatBuff->sz_success_delete_count); + pStorageStat->total_download_count = buff2long( \ + pStatBuff->sz_total_download_count); + pStorageStat->success_download_count = buff2long( \ + pStatBuff->sz_success_download_count); + pStorageStat->total_get_meta_count = buff2long( \ + pStatBuff->sz_total_get_meta_count); + pStorageStat->success_get_meta_count = buff2long( \ + pStatBuff->sz_success_get_meta_count); + pStorageStat->last_source_update = buff2long( \ + pStatBuff->sz_last_source_update); + pStorageStat->last_sync_update = buff2long( \ + pStatBuff->sz_last_sync_update); + pStorageStat->last_synced_timestamp = buff2long( \ + pStatBuff->sz_last_synced_timestamp); + pStorageStat->total_create_link_count = buff2long( \ + pStatBuff->sz_total_create_link_count); + pStorageStat->success_create_link_count = buff2long( \ + pStatBuff->sz_success_create_link_count); + pStorageStat->total_delete_link_count = buff2long( \ + pStatBuff->sz_total_delete_link_count); + pStorageStat->success_delete_link_count = buff2long( \ + pStatBuff->sz_success_delete_link_count); + pStorageStat->total_upload_bytes = buff2long( \ + pStatBuff->sz_total_upload_bytes); + pStorageStat->success_upload_bytes = buff2long( \ + pStatBuff->sz_success_upload_bytes); + pStorageStat->total_append_bytes = buff2long( \ + pStatBuff->sz_total_append_bytes); + pStorageStat->success_append_bytes = buff2long( \ + pStatBuff->sz_success_append_bytes); + pStorageStat->total_modify_bytes = buff2long( \ + pStatBuff->sz_total_modify_bytes); + pStorageStat->success_modify_bytes = buff2long( \ + pStatBuff->sz_success_modify_bytes); + pStorageStat->total_download_bytes = buff2long( \ + pStatBuff->sz_total_download_bytes); + pStorageStat->success_download_bytes = buff2long( \ + pStatBuff->sz_success_download_bytes); + pStorageStat->total_sync_in_bytes = buff2long( \ + pStatBuff->sz_total_sync_in_bytes); + pStorageStat->success_sync_in_bytes = buff2long( \ + pStatBuff->sz_success_sync_in_bytes); + pStorageStat->total_sync_out_bytes = buff2long( \ + pStatBuff->sz_total_sync_out_bytes); + pStorageStat->success_sync_out_bytes = buff2long( \ + pStatBuff->sz_success_sync_out_bytes); + pStorageStat->total_file_open_count = buff2long( \ + pStatBuff->sz_total_file_open_count); + pStorageStat->success_file_open_count = buff2long( \ + pStatBuff->sz_success_file_open_count); + pStorageStat->total_file_read_count = buff2long( \ + pStatBuff->sz_total_file_read_count); + pStorageStat->success_file_read_count = buff2long( \ + pStatBuff->sz_success_file_read_count); + pStorageStat->total_file_write_count = buff2long( \ + pStatBuff->sz_total_file_write_count); + pStorageStat->success_file_write_count = buff2long( \ + pStatBuff->sz_success_file_write_count); + pStorageStat->last_heart_beat_time = buff2long( \ + pStatBuff->sz_last_heart_beat_time); + pDest->if_trunk_server = pSrc->if_trunk_server; + pDest++; + } + + return 0; +} + +int tracker_list_one_group(ConnectionInfo *pTrackerServer, \ + const char *group_name, FDFSGroupStat *pDest) +{ + TrackerHeader *pHeader; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN]; + TrackerGroupStat src; + char *pInBuff; + int result; + int64_t in_bytes; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + pHeader->cmd = TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP; + long2buff(FDFS_GROUP_NAME_MAX_LEN, pHeader->pkg_len); + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = (char *)&src; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(TrackerGroupStat), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes != sizeof(TrackerGroupStat)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + memset(pDest, 0, sizeof(FDFSGroupStat)); + memcpy(pDest->group_name, src.group_name, FDFS_GROUP_NAME_MAX_LEN); + pDest->total_mb = buff2long(src.sz_total_mb); + pDest->free_mb = buff2long(src.sz_free_mb); + pDest->trunk_free_mb = buff2long(src.sz_trunk_free_mb); + pDest->count= buff2long(src.sz_count); + pDest->storage_port= buff2long(src.sz_storage_port); + pDest->storage_http_port= buff2long(src.sz_storage_http_port); + pDest->active_count = buff2long(src.sz_active_count); + pDest->current_write_server = buff2long(src.sz_current_write_server); + pDest->store_path_count = buff2long(src.sz_store_path_count); + pDest->subdir_count_per_path = buff2long(src.sz_subdir_count_per_path); + pDest->current_trunk_file_id = buff2long(src.sz_current_trunk_file_id); + + return 0; +} + +int tracker_list_groups(ConnectionInfo *pTrackerServer, \ + FDFSGroupStat *group_stats, const int max_groups, \ + int *group_count) +{ + bool new_connection; + TrackerHeader header; + TrackerGroupStat stats[FDFS_MAX_GROUPS]; + char *pInBuff; + ConnectionInfo *conn; + TrackerGroupStat *pSrc; + TrackerGroupStat *pEnd; + FDFSGroupStat *pDest; + int result; + int64_t in_bytes; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(&header, 0, sizeof(header)); + header.cmd = TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS; + header.status = 0; + if ((result=tcpsenddata_nb(conn->sock, &header, \ + sizeof(header), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = (char *)stats; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(stats), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + *group_count = 0; + return result; + } + + if (in_bytes % sizeof(TrackerGroupStat) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + *group_count = 0; + return EINVAL; + } + + *group_count = in_bytes / sizeof(TrackerGroupStat); + if (*group_count > max_groups) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d insufficent space, " \ + "max group count: %d, expect count: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, max_groups, *group_count); + *group_count = 0; + return ENOSPC; + } + + memset(group_stats, 0, sizeof(FDFSGroupStat) * max_groups); + pDest = group_stats; + pEnd = stats + (*group_count); + for (pSrc=stats; pSrcgroup_name, pSrc->group_name, \ + FDFS_GROUP_NAME_MAX_LEN); + pDest->total_mb = buff2long(pSrc->sz_total_mb); + pDest->free_mb = buff2long(pSrc->sz_free_mb); + pDest->trunk_free_mb = buff2long(pSrc->sz_trunk_free_mb); + pDest->count= buff2long(pSrc->sz_count); + pDest->storage_port= buff2long(pSrc->sz_storage_port); + pDest->storage_http_port= buff2long(pSrc->sz_storage_http_port); + pDest->active_count = buff2long(pSrc->sz_active_count); + pDest->current_write_server = buff2long( \ + pSrc->sz_current_write_server); + pDest->store_path_count = buff2long( \ + pSrc->sz_store_path_count); + pDest->subdir_count_per_path = buff2long( \ + pSrc->sz_subdir_count_per_path); + pDest->current_trunk_file_id = buff2long( \ + pSrc->sz_current_trunk_file_id); + + pDest++; + } + + return 0; +} + +int tracker_do_query_storage(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const byte cmd, \ + const char *group_name, const char *filename) +{ + TrackerHeader *pHeader; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + 128]; + char in_buff[sizeof(TrackerHeader) + TRACKER_QUERY_STORAGE_FETCH_BODY_LEN]; + char *pInBuff; + int64_t in_bytes; + int result; + int filename_len; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(pStorageServer, 0, sizeof(ConnectionInfo)); + pStorageServer->sock = -1; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + filename_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", filename); + + long2buff(FDFS_GROUP_NAME_MAX_LEN + filename_len, pHeader->pkg_len); + pHeader->cmd = cmd; + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + + filename_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = in_buff; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(in_buff), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes != TRACKER_QUERY_STORAGE_FETCH_BODY_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length: %d", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes, \ + TRACKER_QUERY_STORAGE_FETCH_BODY_LEN); + return EINVAL; + } + + memcpy(pStorageServer->ip_addr, in_buff + \ + FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE-1); + pStorageServer->port = (int)buff2long(in_buff + \ + FDFS_GROUP_NAME_MAX_LEN + IP_ADDRESS_SIZE - 1); + return 0; +} + +int tracker_query_storage_list(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int nMaxServerCount, \ + int *server_count, char *group_name, const char *filename) +{ + TrackerHeader *pHeader; + ConnectionInfo *pServer; + ConnectionInfo *pServerEnd; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + 128]; + char in_buff[sizeof(TrackerHeader) + \ + TRACKER_QUERY_STORAGE_FETCH_BODY_LEN + \ + FDFS_MAX_SERVERS_EACH_GROUP * IP_ADDRESS_SIZE]; + char *pInBuff; + int64_t in_bytes; + int result; + int filename_len; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + filename_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", filename); + + long2buff(FDFS_GROUP_NAME_MAX_LEN + filename_len, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL; + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + + filename_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = in_buff; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(in_buff), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if ((in_bytes - TRACKER_QUERY_STORAGE_FETCH_BODY_LEN) % \ + (IP_ADDRESS_SIZE - 1) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + *server_count = 1 + (in_bytes - TRACKER_QUERY_STORAGE_FETCH_BODY_LEN) / + (IP_ADDRESS_SIZE - 1); + if (nMaxServerCount < *server_count) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response storage server " \ + "count: %d, exceeds max server count: %d!", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + *server_count, nMaxServerCount); + return ENOSPC; + } + + memset(pStorageServer, 0, nMaxServerCount * sizeof(ConnectionInfo)); + pStorageServer->sock = -1; + + memcpy(group_name, pInBuff, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + pInBuff += FDFS_GROUP_NAME_MAX_LEN; + memcpy(pStorageServer->ip_addr, pInBuff, IP_ADDRESS_SIZE - 1); + pInBuff += IP_ADDRESS_SIZE - 1; + pStorageServer->port = (int)buff2long(pInBuff); + pInBuff += FDFS_PROTO_PKG_LEN_SIZE; + + pServerEnd = pStorageServer + (*server_count); + for (pServer=pStorageServer+1; pServersock = -1; + pServer->port = pStorageServer->port; + memcpy(pServer->ip_addr, pInBuff, IP_ADDRESS_SIZE - 1); + pInBuff += IP_ADDRESS_SIZE - 1; + } + + return 0; +} + +int tracker_query_storage_store_without_group(ConnectionInfo *pTrackerServer, + ConnectionInfo *pStorageServer, char *group_name, + int *store_path_index) +{ + TrackerHeader header; + char in_buff[sizeof(TrackerHeader) + \ + TRACKER_QUERY_STORAGE_STORE_BODY_LEN]; + bool new_connection; + ConnectionInfo *conn; + char *pInBuff; + int64_t in_bytes; + int result; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(pStorageServer, 0, sizeof(ConnectionInfo)); + pStorageServer->sock = -1; + + memset(&header, 0, sizeof(header)); + header.cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE; + if ((result=tcpsenddata_nb(conn->sock, &header, \ + sizeof(header), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = in_buff; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(in_buff), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes != TRACKER_QUERY_STORAGE_STORE_BODY_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length: %d", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + in_bytes, TRACKER_QUERY_STORAGE_STORE_BODY_LEN); + return EINVAL; + } + + memcpy(group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + memcpy(pStorageServer->ip_addr, in_buff + \ + FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE-1); + pStorageServer->port = (int)buff2long(in_buff + \ + FDFS_GROUP_NAME_MAX_LEN + IP_ADDRESS_SIZE - 1); + *store_path_index = *(in_buff + FDFS_GROUP_NAME_MAX_LEN + \ + IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE); + + return 0; +} + +int tracker_query_storage_store_with_group(ConnectionInfo *pTrackerServer, \ + const char *group_name, ConnectionInfo *pStorageServer, \ + int *store_path_index) +{ + TrackerHeader *pHeader; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN]; + char in_buff[sizeof(TrackerHeader) + \ + TRACKER_QUERY_STORAGE_STORE_BODY_LEN]; + char *pInBuff; + int64_t in_bytes; + int result; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + memset(pStorageServer, 0, sizeof(ConnectionInfo)); + pStorageServer->sock = -1; + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + + long2buff(FDFS_GROUP_NAME_MAX_LEN, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE; + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN, \ + g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = in_buff; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(in_buff), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes != TRACKER_QUERY_STORAGE_STORE_BODY_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length: %d", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + in_bytes, TRACKER_QUERY_STORAGE_STORE_BODY_LEN); + return EINVAL; + } + + memcpy(pStorageServer->ip_addr, in_buff + \ + FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE-1); + pStorageServer->port = (int)buff2long(in_buff + \ + FDFS_GROUP_NAME_MAX_LEN + IP_ADDRESS_SIZE - 1); + *store_path_index = *(in_buff + FDFS_GROUP_NAME_MAX_LEN + \ + IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE); + + return 0; +} + +int tracker_query_storage_store_list_with_group( \ + ConnectionInfo *pTrackerServer, const char *group_name, \ + ConnectionInfo *storageServers, const int nMaxServerCount, \ + int *storage_count, int *store_path_index) +{ + ConnectionInfo *pStorageServer; + ConnectionInfo *pServerEnd; + TrackerHeader *pHeader; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN]; + char in_buff[sizeof(TrackerHeader) + FDFS_MAX_SERVERS_EACH_GROUP * \ + TRACKER_QUERY_STORAGE_STORE_BODY_LEN]; + char returned_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *pInBuff; + char *p; + int64_t in_bytes; + int out_len; + int ipPortsLen; + int result; + + *storage_count = 0; + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + + if (group_name == NULL || *group_name == '\0') + { + pHeader->cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL; + out_len = 0; + } + else + { + pHeader->cmd = TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL; + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + out_len = FDFS_GROUP_NAME_MAX_LEN; + } + + long2buff(out_len, pHeader->pkg_len); + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(TrackerHeader) + out_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = in_buff; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(in_buff), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes < TRACKER_QUERY_STORAGE_STORE_BODY_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length >= %d", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + in_bytes, TRACKER_QUERY_STORAGE_STORE_BODY_LEN); + return EINVAL; + } + +#define RECORD_LENGTH (IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE) + + ipPortsLen = in_bytes - (FDFS_GROUP_NAME_MAX_LEN + 1); + if (ipPortsLen % RECORD_LENGTH != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + *storage_count = ipPortsLen / RECORD_LENGTH; + if (nMaxServerCount < *storage_count) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response storage server " \ + "count: %d, exceeds max server count: %d!", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, *storage_count, nMaxServerCount); + return ENOSPC; + } + + memset(storageServers, 0, sizeof(ConnectionInfo) * nMaxServerCount); + + memcpy(returned_group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + p = in_buff + FDFS_GROUP_NAME_MAX_LEN; + *(returned_group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + + pServerEnd = storageServers + (*storage_count); + for (pStorageServer=storageServers; pStorageServersock = -1; + memcpy(pStorageServer->ip_addr, p, IP_ADDRESS_SIZE - 1); + p += IP_ADDRESS_SIZE - 1; + + pStorageServer->port = (int)buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + } + + *store_path_index = *p; + + return 0; +} + +int tracker_delete_storage(TrackerServerGroup *pTrackerGroup, \ + const char *group_name, const char *storage_id) +{ + ConnectionInfo *conn; + TrackerHeader *pHeader; + ConnectionInfo tracker_server; + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + FDFSStorageInfo storage_infos[1]; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_STORAGE_ID_MAX_SIZE]; + char in_buff[1]; + char *pInBuff; + int64_t in_bytes; + int result; + int storage_id_len; + int storage_count; + int enoent_count; + + enoent_count = 0; + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pTrackerGroup->servers; pServerserver_count) + { + return ENOENT; + } + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + storage_id_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", storage_id); + + long2buff(FDFS_GROUP_NAME_MAX_LEN + storage_id_len, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_SERVER_DELETE_STORAGE; + + enoent_count = 0; + result = 0; + for (pServer=pTrackerGroup->servers; pServersock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + + storage_id_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + tracker_server.ip_addr, tracker_server.port, \ + result, STRERROR(result)); + } + else + { + pInBuff = in_buff; + result = fdfs_recv_response(conn, &pInBuff, 0, &in_bytes); + } + + tracker_disconnect_server_ex(conn, result != 0 && result != ENOENT); + if (result != 0) + { + if (result == ENOENT) + { + enoent_count++; + } + else if (result == EALREADY) + { + } + else + { + return result; + } + } + } + + if (enoent_count == pTrackerGroup->server_count) + { + return ENOENT; + } + + return result == ENOENT ? 0 : result; +} + +int tracker_set_trunk_server(TrackerServerGroup *pTrackerGroup, \ + const char *group_name, const char *storage_id, \ + char *new_trunk_server_id) +{ + TrackerHeader *pHeader; + ConnectionInfo *conn; + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + ConnectionInfo tracker_server; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_STORAGE_ID_MAX_SIZE]; + char in_buff[FDFS_STORAGE_ID_MAX_SIZE]; + char *pInBuff; + int64_t in_bytes; + int result; + int storage_id_len; + + *new_trunk_server_id = '\0'; + memset(out_buff, 0, sizeof(out_buff)); + memset(in_buff, 0, sizeof(in_buff)); + pHeader = (TrackerHeader *)out_buff; + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", group_name); + if (storage_id == NULL) + { + storage_id_len = 0; + } + else + { + storage_id_len = snprintf(out_buff + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, \ + sizeof(out_buff) - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN, "%s", storage_id); + } + + long2buff(FDFS_GROUP_NAME_MAX_LEN + storage_id_len, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_SERVER_SET_TRUNK_SERVER; + + result = 0; + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pTrackerGroup->servers; pServersock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + + storage_id_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + tracker_server.ip_addr, tracker_server.port, \ + result, STRERROR(result)); + + tracker_disconnect_server_ex(conn, true); + continue; + } + + pInBuff = in_buff; + result = fdfs_recv_response(conn, &pInBuff, \ + sizeof(in_buff) - 1, &in_bytes); + + tracker_disconnect_server_ex(conn, result != 0); + if (result == 0) + { + strcpy(new_trunk_server_id, in_buff); + return 0; + } + + if (result == EOPNOTSUPP) + { + continue; + } + if (result == EALREADY) + { + if (storage_id_len > 0) + { + strcpy(new_trunk_server_id, storage_id); + } + return result; + } + else + { + return result; + } + } + + return result; +} + +int tracker_get_storage_status(ConnectionInfo *pTrackerServer, \ + const char *group_name, const char *ip_addr, \ + FDFSStorageBrief *pDestBuff) +{ + TrackerHeader *pHeader; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + IP_ADDRESS_SIZE]; + char *pInBuff; + char *p; + int result; + int ip_len; + int64_t in_bytes; + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + if (ip_addr == NULL) + { + ip_len = 0; + } + else + { + ip_len = strlen(ip_addr); + } + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + snprintf(p, sizeof(out_buff) - sizeof(TrackerHeader), "%s", group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + if (ip_len > 0) + { + memcpy(p, ip_addr, ip_len); + p += ip_len; + } + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_GET_STATUS; + long2buff(FDFS_GROUP_NAME_MAX_LEN + ip_len, pHeader->pkg_len); + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + pInBuff = (char *)pDestBuff; + result = fdfs_recv_response(conn, \ + &pInBuff, sizeof(FDFSStorageBrief), &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes != sizeof(FDFSStorageBrief)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +int tracker_get_storage_id(ConnectionInfo *pTrackerServer, \ + const char *group_name, const char *ip_addr, \ + char *storage_id) +{ + TrackerHeader *pHeader; + ConnectionInfo *conn; + bool new_connection; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + IP_ADDRESS_SIZE]; + char *p; + int result; + int ip_len; + int64_t in_bytes; + + if (storage_id == NULL) + { + return EINVAL; + } + + CHECK_CONNECTION(pTrackerServer, conn, result, new_connection); + + if (ip_addr == NULL) + { + ip_len = 0; + } + else + { + ip_len = strlen(ip_addr); + } + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + snprintf(p, sizeof(out_buff) - sizeof(TrackerHeader), "%s", group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + if (ip_len > 0) + { + memcpy(p, ip_addr, ip_len); + p += ip_len; + } + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_GET_SERVER_ID; + long2buff(FDFS_GROUP_NAME_MAX_LEN + ip_len, pHeader->pkg_len); + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + result = fdfs_recv_response(conn, \ + &storage_id, FDFS_STORAGE_ID_MAX_SIZE, &in_bytes); + } + + if (new_connection) + { + tracker_disconnect_server_ex(conn, result != 0); + } + + if (result != 0) + { + return result; + } + + if (in_bytes == 0 || in_bytes >= FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + *(storage_id + in_bytes) = '\0'; + return 0; +} + +int tracker_get_storage_max_status(TrackerServerGroup *pTrackerGroup, \ + const char *group_name, const char *ip_addr, \ + char *storage_id, int *status) +{ + ConnectionInfo *conn; + ConnectionInfo tracker_server; + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + FDFSStorageBrief storage_brief; + int result; + + memset(&storage_brief, 0, sizeof(FDFSStorageBrief)); + storage_brief.status = -1; + + *storage_id = '\0'; + *status = -1; + pEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + for (pServer=pTrackerGroup->servers; pServer *status) + { + *status = storage_brief.status; + } + } + + if (*status == -1) + { + return ENOENT; + } + + return 0; +} + diff --git a/client/tracker_client.h b/client/tracker_client.h new file mode 100644 index 0000000..fb11a10 --- /dev/null +++ b/client/tracker_client.h @@ -0,0 +1,353 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef TRACKER_CLIENT_H +#define TRACKER_CLIENT_H + +#include "tracker_types.h" +#include "tracker_proto.h" +#include "client_global.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct +{ + bool if_trunk_server; + char status; + char id[FDFS_STORAGE_ID_MAX_SIZE]; + char ip_addr[IP_ADDRESS_SIZE]; + char src_id[FDFS_STORAGE_ID_MAX_SIZE]; //src storage id + char domain_name[FDFS_DOMAIN_NAME_MAX_SIZE]; //http domain name + char version[FDFS_VERSION_SIZE]; + int total_mb; //total disk storage in MB + int free_mb; //free disk storage in MB + int upload_priority; //upload priority + time_t join_time; //storage join timestamp (create timestamp) + time_t up_time; //storage service started timestamp + int store_path_count; //store base path count of each storage server + int subdir_count_per_path; + int storage_port; + int storage_http_port; //storage http server port + int current_write_path; //current write path index + FDFSStorageStat stat; +} FDFSStorageInfo; + + +#define CHECK_CONNECTION(pTrackerServer, conn, result, new_connection) \ + do { \ + if (pTrackerServer->sock < 0) \ + { \ + if ((conn=tracker_connect_server( \ + pTrackerServer, &result)) != NULL) \ + { \ + return result; \ + } \ + new_connection = true; \ + } \ + else \ + { \ + conn = pTrackerServer; \ + new_connection = false; \ + } \ + } while (0) + + +#define tracker_get_connection() \ + tracker_get_connection_ex((&g_tracker_group)) + +/** +* get a connection to tracker server +* params: +* pTrackerGroup: the tracker group +* return: != NULL for success, NULL for fail +**/ +ConnectionInfo *tracker_get_connection_ex(TrackerServerGroup *pTrackerGroup); + + +#define tracker_get_connection_r(pTrackerServer, err_no) \ + tracker_get_connection_r_ex((&g_tracker_group), pTrackerServer, err_no) + +/** +* get a connection to tracker server +* params: +* pTrackerGroup: the tracker group +* pTrackerServer: tracker server +* return: 0 success, !=0 fail +**/ +ConnectionInfo *tracker_get_connection_r_ex(TrackerServerGroup *pTrackerGroup, \ + ConnectionInfo *pTrackerServer, int *err_no); + +#define tracker_get_all_connections() \ + tracker_get_all_connections_ex((&g_tracker_group)) + + +/** +* get a connection to tracker server without connection pool +* params: +* pTrackerGroup: the tracker group +* return: != NULL for success, NULL for fail +**/ +ConnectionInfo *tracker_get_connection_no_pool( \ + TrackerServerGroup *pTrackerGroup); + +/** +* connect to all tracker servers +* params: +* pTrackerGroup: the tracker group +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_get_all_connections_ex(TrackerServerGroup *pTrackerGroup); + +#define tracker_close_all_connections() \ + tracker_close_all_connections_ex((&g_tracker_group)) + +/** +* close all connections to tracker servers +* params: +* pTrackerGroup: the tracker group +* return: +**/ +void tracker_close_all_connections_ex(TrackerServerGroup *pTrackerGroup); + +/** +* list one group +* params: +* pTrackerServer: tracker server +* group_name: the group name +* pDest: return the group info +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_list_one_group(ConnectionInfo *pTrackerServer, \ + const char *group_name, FDFSGroupStat *pDest); + + +/** +* list all groups +* params: +* pTrackerServer: tracker server +* group_stats: return group info array +* max_groups: max group count(group array capacity) +* group_count: return group count +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_list_groups(ConnectionInfo *pTrackerServer, \ + FDFSGroupStat *group_stats, const int max_groups, \ + int *group_count); + +/** +* list all servers of the specified group +* params: +* pTrackerServer: tracker server +* szGroupName: group name to query +* szStorageId: the storage id to query, can be NULL or empty +* storage_infos: return storage info array +* max_storages: max storage count(storage array capacity) +* storage_count: return storage count +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_list_servers(ConnectionInfo *pTrackerServer, \ + const char *szGroupName, const char *szStorageId, \ + FDFSStorageInfo *storage_infos, const int max_storages, \ + int *storage_count); + +#define tracker_query_storage_store(pTrackerServer, pStorageServer, \ + group_name, store_path_index) \ + tracker_query_storage_store_without_group(pTrackerServer, \ + pStorageServer, group_name, store_path_index) +/** +* query storage server to upload file +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* store_path_index: return the index of path on the storage server +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_store_without_group(ConnectionInfo *pTrackerServer, + ConnectionInfo *pStorageServer, char *group_name, + int *store_path_index); + +/** +* query storage servers/list to upload file +* params: +* pTrackerServer: tracker server +* storageServers: store the storage server list +* nMaxServerCount: max storage server count +* storage_count: return the storage server count +* store_path_index: return the index of path on the storage server +* return: 0 success, !=0 fail, return the error code +**/ +#define tracker_query_storage_store_list_without_group( \ + pTrackerServer, storageServers, nMaxServerCount, \ + storage_count, group_name, store_path_index) \ + tracker_query_storage_store_list_with_group( \ + pTrackerServer, NULL, storageServers, nMaxServerCount, \ + storage_count, store_path_index) + +/** +* query storage server to upload file +* params: +* pTrackerServer: tracker server +* group_name: the group name to upload file to +* pStorageServer: return storage server +* store_path_index: return the index of path on the storage server +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_store_with_group(ConnectionInfo *pTrackerServer, \ + const char *group_name, ConnectionInfo *pStorageServer, \ + int *store_path_index); + +/** +* query storage servers/list to upload file +* params: +* pTrackerServer: tracker server +* group_name: the group name to upload file to +* storageServers: store the storage server list +* nMaxServerCount: max storage server count +* storage_count: return the storage server count +* store_path_index: return the index of path on the storage server +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_store_list_with_group( \ + ConnectionInfo *pTrackerServer, const char *group_name, \ + ConnectionInfo *storageServers, const int nMaxServerCount, \ + int *storage_count, int *store_path_index); + +/** +* query storage server to update (delete file or set meta data) +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* group_name: the group name of storage server +* filename: filename on storage server +* return: 0 success, !=0 fail, return the error code +**/ +#define tracker_query_storage_update(pTrackerServer, \ + pStorageServer, group_name, filename) \ + tracker_do_query_storage(pTrackerServer, \ + pStorageServer, TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE,\ + group_name, filename) + +/** +* query storage server to download file +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* group_name: the group name of storage server +* filename: filename on storage server +* return: 0 success, !=0 fail, return the error code +**/ +#define tracker_query_storage_fetch(pTrackerServer, \ + pStorageServer, group_name, filename) \ + tracker_do_query_storage(pTrackerServer, \ + pStorageServer, TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE,\ + group_name, filename) + +/** +* query storage server to fetch or update +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* cmd : command, TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE or +* TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE +* group_name: the group name of storage server +* filename: filename on storage server +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_do_query_storage(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const byte cmd, \ + const char *group_name, const char *filename); + +/** +* query storage server list to fetch file +* params: +* pTrackerServer: tracker server +* pStorageServer: return storage server +* nMaxServerCount: max storage server count +* server_count: return storage server count +* group_name: the group name of storage server +* filename: filename on storage server +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_query_storage_list(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const int nMaxServerCount, \ + int *server_count, char *group_name, const char *filename); + +/** +* delete a storage server from cluster +* params: +* pTrackerGroup: the tracker group +* group_name: the group name which the storage server belongs to +* storage_id: the storage server id +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_delete_storage(TrackerServerGroup *pTrackerGroup, \ + const char *group_name, const char *storage_id); + + +/** +* set trunk server of the specified group +* params: +* pTrackerGroup: the tracker group +* group_name: the group name which the storage server belongs to +* storage_id: the storage server id, can be NULL or empty +* new_trunk_server_id: the new trunk server id +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_set_trunk_server(TrackerServerGroup *pTrackerGroup, \ + const char *group_name, const char *storage_id, \ + char *new_trunk_server_id); + + +/** +* get storage server status from the tracker server +* params: +* pTrackerServer: tracker server +* group_name: the group name which the storage server belongs to +* ip_addr: the ip addr of the storage server +* pDestBuff: return the storage server brief info +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_get_storage_status(ConnectionInfo *pTrackerServer, \ + const char *group_name, const char *ip_addr, \ + FDFSStorageBrief *pDestBuff); + + +/** +* get storage server id from the tracker server +* params: +* pTrackerServer: tracker server +* group_name: the group name which the storage server belongs to +* ip_addr: the ip addr of the storage server +* storage_id: return the storage server id +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_get_storage_id(ConnectionInfo *pTrackerServer, \ + const char *group_name, const char *ip_addr, \ + char *storage_id); + +/** +* get storage server highest level status from all tracker servers +* params: +* pTrackerGroup: the tracker group +* group_name: the group name which the storage server belongs to +* ip_addr: the ip addr of the storage server +* storage_id: return the storage server id +* status: return the highest level status +* return: 0 success, !=0 fail, return the error code +**/ +int tracker_get_storage_max_status(TrackerServerGroup *pTrackerGroup, \ + const char *group_name, const char *ip_addr, \ + char *storage_id, int *status); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/common/Makefile b/common/Makefile new file mode 100644 index 0000000..c7ffc92 --- /dev/null +++ b/common/Makefile @@ -0,0 +1,28 @@ +.SUFFIXES: .c .o + +COMPILE = $(CC) -Wall -O2 -D_FILE_OFFSET_BITS=64 -DOS_LINUX +#COMPILE = $(CC) -Wall -g -D_FILE_OFFSET_BITS=64 -DOS_LINUX -D__DEBUG__ +INC_PATH = -I/usr/local/include +LIB_PATH = -L/usr/local/lib +TARGET_PATH = /usr/local/bin + +COMMON_LIB = +SHARED_OBJS = hash.o chain.o shared_func.o ini_file_reader.o \ + logger.o sockopt.o fdfs_global.o base64.o sched_thread.o \ + mime_file_parser.o fdfs_http_shared.o + +ALL_OBJS = $(SHARED_OBJS) + +ALL_PRGS = + +all: $(ALL_OBJS) $(ALL_PRGS) +.o: + $(COMPILE) -o $@ $< $(SHARED_OBJS) $(COMMON_LIB) $(LIB_PATH) $(INC_PATH) +.c: + $(COMPILE) -o $@ $< $(ALL_OBJS) $(COMMON_LIB) $(LIB_PATH) $(INC_PATH) +.c.o: + $(COMPILE) -c -o $@ $< $(INC_PATH) +install: + cp -f $(ALL_PRGS) $(TARGET_PATH) +clean: + rm -f $(ALL_OBJS) $(ALL_PRGS) diff --git a/common/_os_bits.h b/common/_os_bits.h new file mode 100644 index 0000000..81e1626 --- /dev/null +++ b/common/_os_bits.h @@ -0,0 +1,7 @@ +#ifndef _OS_BITS_H +#define _OS_BITS_H + +#define OS_BITS 64 +#define OFF_BITS 64 + +#endif diff --git a/common/avl_tree.c b/common/avl_tree.c new file mode 100644 index 0000000..65c7e36 --- /dev/null +++ b/common/avl_tree.c @@ -0,0 +1,789 @@ +#include "avl_tree.h" + +int avl_tree_init(AVLTreeInfo *tree, FreeDataFunc free_data_func, \ + CompareFunc compare_func) +{ + tree->root = NULL; + tree->free_data_func = free_data_func; + tree->compare_func = compare_func; + return 0; +} + +static void avl_tree_destroy_loop(FreeDataFunc free_data_func, \ + AVLTreeNode *pCurrentNode) +{ + if (pCurrentNode->left != NULL) + { + avl_tree_destroy_loop(free_data_func, pCurrentNode->left); + } + if (pCurrentNode->right != NULL) + { + avl_tree_destroy_loop(free_data_func, pCurrentNode->right); + } + + if (free_data_func != NULL) + { + free_data_func(pCurrentNode->data); + } + free(pCurrentNode); +} + +void avl_tree_destroy(AVLTreeInfo *tree) +{ + if (tree == NULL) + { + return; + } + + if (tree->root != NULL) + { + avl_tree_destroy_loop(tree->free_data_func, tree->root); + tree->root = NULL; + } +} + +static AVLTreeNode *createTreeNode(AVLTreeNode *pParentNode, void *target_data) +{ + AVLTreeNode *pNewNode; + pNewNode = (AVLTreeNode *)malloc(sizeof(AVLTreeNode)); + if (pNewNode == NULL) + { + /* + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "malloc %d bytes fail!\n", __LINE__, \ + (int)sizeof(AVLTreeNode)); + */ + return NULL; + } + + pNewNode->left = pNewNode->right = NULL; + pNewNode->data = target_data; + pNewNode->balance = 0; + + return pNewNode; +} + +static void avlRotateLeft(AVLTreeNode *pRotateNode, AVLTreeNode **ppRaiseNode) +{ + *ppRaiseNode = pRotateNode->right; + pRotateNode->right = (*ppRaiseNode)->left; + (*ppRaiseNode)->left = pRotateNode; +} + +static void avlRotateRight(AVLTreeNode *pRotateNode, AVLTreeNode **ppRaiseNode) +{ + *ppRaiseNode = pRotateNode->left; + pRotateNode->left = (*ppRaiseNode)->right; + (*ppRaiseNode)->right = pRotateNode; +} + +static void avlLeftBalanceWhenInsert(AVLTreeNode **pTreeNode, int *taller) +{ + AVLTreeNode *leftsub; + AVLTreeNode *rightsub; + + leftsub = (*pTreeNode)->left; + switch (leftsub->balance) + { + case -1 : + (*pTreeNode)->balance = leftsub->balance = 0; + avlRotateRight (*pTreeNode, pTreeNode); + *taller = 0; + break; + case 0 : + break; + case 1 : + rightsub = leftsub->right; + switch ( rightsub->balance ) + { + case -1: + (*pTreeNode)->balance = 1; + leftsub->balance = 0; + break; + case 0 : + (*pTreeNode)->balance = leftsub->balance = 0; + break; + case 1 : + (*pTreeNode)->balance = 0; + leftsub->balance = -1; + break; + } + + rightsub->balance = 0; + avlRotateLeft( leftsub, &((*pTreeNode)->left)); + avlRotateRight(*pTreeNode, pTreeNode); + *taller = 0; + } +} + +static void avlRightBalanceWhenInsert(AVLTreeNode **pTreeNode, int *taller) +{ + AVLTreeNode *rightsub; + AVLTreeNode *leftsub; + + rightsub = (*pTreeNode)->right; + switch (rightsub->balance) + { + case 1: + (*pTreeNode)->balance = rightsub->balance = 0; + avlRotateLeft(*pTreeNode, pTreeNode); + *taller = 0; + break; + case 0: + break; + case -1: + leftsub = rightsub->left; + switch (leftsub->balance) + { + case 1 : + (*pTreeNode)->balance = -1; + rightsub->balance = 0; + break; + case 0 : + (*pTreeNode)->balance = rightsub->balance = 0; + break; + case -1 : + (*pTreeNode)->balance = 0; + rightsub->balance = 1; + break; + } + + leftsub->balance = 0; + avlRotateRight(rightsub, &((*pTreeNode)->right)); + avlRotateLeft(*pTreeNode, pTreeNode); + *taller = 0; + } +} + +static int avl_tree_insert_loop(CompareFunc compare_func, AVLTreeNode **pCurrentNode, \ + void *target_data, int *taller) +{ + int nCompRes; + int success; + + if (*pCurrentNode == NULL) + { + *pCurrentNode = createTreeNode(*pCurrentNode, target_data); + if (*pCurrentNode != NULL) + { + *taller = 1; + return 1; + } + else + { + return -ENOMEM; + } + } + + nCompRes = compare_func((*pCurrentNode)->data, target_data); + if (nCompRes > 0) + { + success = avl_tree_insert_loop(compare_func, \ + &((*pCurrentNode)->left), target_data, taller); + if (*taller != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1: + avlLeftBalanceWhenInsert(pCurrentNode, taller); + break; + case 0: + (*pCurrentNode)->balance = -1; + break; + case 1: + (*pCurrentNode)->balance = 0; + *taller = 0; + break; + } + } + } + else if (nCompRes < 0) + { + success = avl_tree_insert_loop(compare_func, \ + &((*pCurrentNode)->right), target_data, taller); + if (*taller != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1: + (*pCurrentNode)->balance = 0; + *taller = 0; + break; + case 0: + (*pCurrentNode)->balance = 1; + break; + case 1: + avlRightBalanceWhenInsert(pCurrentNode, taller); + break; + } + } + } + else + { + return 0; + } + + return success; +} + +int avl_tree_insert(AVLTreeInfo *tree, void *data) +{ + int taller; + + taller = 0; + return avl_tree_insert_loop(tree->compare_func, &(tree->root), \ + data, &taller); +} + +static int avl_tree_replace_loop(CompareFunc compare_func, \ + FreeDataFunc free_data_func, AVLTreeNode **pCurrentNode, \ + void *target_data, int *taller) +{ + int nCompRes; + int success; + + if (*pCurrentNode == NULL ) + { + *pCurrentNode = createTreeNode(*pCurrentNode, target_data); + if (*pCurrentNode != NULL) + { + *taller = 1; + return 1; + } + else + { + return -ENOMEM; + } + } + + nCompRes = compare_func((*pCurrentNode)->data, target_data); + if (nCompRes > 0) + { + success = avl_tree_replace_loop(compare_func, free_data_func, + &((*pCurrentNode)->left), target_data, taller); + if (*taller != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1: + avlLeftBalanceWhenInsert(pCurrentNode, taller); + break; + case 0: + (*pCurrentNode)->balance = -1; + break; + case 1: + (*pCurrentNode)->balance = 0; + *taller = 0; + break; + } + } + } + else if (nCompRes < 0) + { + success = avl_tree_replace_loop(compare_func, free_data_func, + &((*pCurrentNode)->right), target_data, taller); + if (*taller != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1 : + (*pCurrentNode)->balance = 0; + *taller = 0; + break; + case 0 : + (*pCurrentNode)->balance = 1; + break; + case 1 : + avlRightBalanceWhenInsert(pCurrentNode, taller); + break; + } + } + } + else + { + if (free_data_func != NULL) + { + free_data_func((*pCurrentNode)->data); + } + (*pCurrentNode)->data = target_data; + return 0; + } + + return success; +} + +int avl_tree_replace(AVLTreeInfo *tree, void *data) +{ + int taller; + + taller = 0; + return avl_tree_replace_loop(tree->compare_func, \ + tree->free_data_func, &(tree->root), data, &taller); +} + +static AVLTreeNode *avl_tree_find_loop(CompareFunc compare_func, \ + AVLTreeNode *pCurrentNode, void *target_data) +{ + int nCompRes; + nCompRes = compare_func(pCurrentNode->data, target_data); + if (nCompRes > 0) + { + if (pCurrentNode->left == NULL) + { + return NULL; + } + else + { + return avl_tree_find_loop(compare_func, \ + pCurrentNode->left, target_data); + } + } + else if (nCompRes < 0) + { + if (pCurrentNode->right == NULL) + { + return NULL; + } + else + { + return avl_tree_find_loop(compare_func, \ + pCurrentNode->right, target_data); + } + } + else + { + return pCurrentNode; + } +} + +static void *avl_tree_find_ge_loop(CompareFunc compare_func, \ + AVLTreeNode *pCurrentNode, void *target_data) +{ + int nCompRes; + void *found; + + nCompRes = compare_func(pCurrentNode->data, target_data); + if (nCompRes > 0) + { + if (pCurrentNode->left == NULL) + { + return pCurrentNode->data; + } + + found = avl_tree_find_ge_loop(compare_func, \ + pCurrentNode->left, target_data); + return found != NULL ? found : pCurrentNode->data; + } + else if (nCompRes < 0) + { + if (pCurrentNode->right == NULL) + { + return NULL; + } + else + { + return avl_tree_find_ge_loop(compare_func, \ + pCurrentNode->right, target_data); + } + } + else + { + return pCurrentNode->data; + } +} + +void *avl_tree_find(AVLTreeInfo *tree, void *target_data) +{ + AVLTreeNode *found; + + if (tree->root == NULL) + { + return NULL; + } + + found = avl_tree_find_loop(tree->compare_func, \ + tree->root, target_data); + + return found != NULL ? found->data : NULL; +} + +void *avl_tree_find_ge(AVLTreeInfo *tree, void *target_data) +{ + void *found; + + if (tree->root == NULL) + { + found = NULL; + } + else + { + found = avl_tree_find_ge_loop(tree->compare_func, \ + tree->root, target_data); + } + + return found; +} + +static void avlLeftBalanceWhenDelete(AVLTreeNode **pTreeNode, int *shorter) +{ + AVLTreeNode *leftsub; + AVLTreeNode *rightsub; + + leftsub = (*pTreeNode)->left; + switch (leftsub->balance) + { + case -1: + (*pTreeNode)->balance = leftsub->balance = 0; + avlRotateRight (*pTreeNode, pTreeNode); + break; + case 0: + leftsub->balance = 1; + avlRotateRight(*pTreeNode, pTreeNode); + *shorter = 0; + break; + case 1: + rightsub = leftsub->right; + switch ( rightsub->balance ) + { + case -1: + (*pTreeNode)->balance = 1; + leftsub->balance = 0; + break; + case 0 : + (*pTreeNode)->balance = leftsub->balance = 0; + break; + case 1 : + (*pTreeNode)->balance = 0; + leftsub->balance = -1; + break; + } + + rightsub->balance = 0; + avlRotateLeft( leftsub, &((*pTreeNode)->left)); + avlRotateRight(*pTreeNode, pTreeNode); + break; + } +} + +static void avlRightBalanceWhenDelete(AVLTreeNode **pTreeNode, int *shorter) +{ + AVLTreeNode *rightsub; + AVLTreeNode *leftsub; + + rightsub = (*pTreeNode)->right; + switch (rightsub->balance) + { + case 1: + (*pTreeNode)->balance = rightsub->balance = 0; + avlRotateLeft(*pTreeNode, pTreeNode); + break; + case 0: + rightsub->balance = -1; + avlRotateLeft(*pTreeNode, pTreeNode); + *shorter = 0; + break; + case -1: + leftsub = rightsub->left; + switch (leftsub->balance) + { + case 1: + (*pTreeNode)->balance = -1; + rightsub->balance = 0; + break; + case 0: + (*pTreeNode)->balance = rightsub->balance = 0; + break; + case -1: + (*pTreeNode)->balance = 0; + rightsub->balance = 1; + break; + } + + leftsub->balance = 0; + avlRotateRight(rightsub, &((*pTreeNode)->right)); + avlRotateLeft(*pTreeNode, pTreeNode); + break; + } +} + +static int avl_tree_delete_loop(AVLTreeInfo *tree, AVLTreeNode **pCurrentNode,\ + void *target_data, int *shorter, AVLTreeNode *pDeletedDataNode) +{ + int nCompRes; + bool result; + AVLTreeNode *leftsub; + AVLTreeNode *rightsub; + + if (pDeletedDataNode != NULL) + { + if ((*pCurrentNode)->right == NULL) + { + pDeletedDataNode->data = (*pCurrentNode)->data; + leftsub = (*pCurrentNode)->left; + + free(*pCurrentNode); + *pCurrentNode = leftsub; + *shorter = 1; + return 1; + } + + nCompRes = -1; + } + else + { + nCompRes = tree->compare_func((*pCurrentNode)->data, target_data); + } + + if (nCompRes > 0) + { + if ((*pCurrentNode)->left == NULL) + { + return 0; + } + + result = avl_tree_delete_loop(tree, &((*pCurrentNode)->left), \ + target_data, shorter, pDeletedDataNode); + if (*shorter != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1: + (*pCurrentNode)->balance = 0; + break; + case 0: + (*pCurrentNode)->balance = 1; + *shorter = 0; + break; + case 1: + avlRightBalanceWhenDelete(pCurrentNode, shorter); + break; + } + } + return result; + } + else if (nCompRes < 0) + { + if ((*pCurrentNode)->right == NULL) + { + return 0; + } + + result = avl_tree_delete_loop(tree, &((*pCurrentNode)->right), \ + target_data, shorter, pDeletedDataNode); + if (*shorter != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1: + avlLeftBalanceWhenDelete(pCurrentNode, shorter); + break; + case 0: + (*pCurrentNode)->balance = -1; + *shorter = 0; + break; + case 1: + (*pCurrentNode)->balance = 0; + break; + } + } + return result; + } + else + { + if (tree->free_data_func != NULL) + { + tree->free_data_func((*pCurrentNode)->data); + } + + leftsub = (*pCurrentNode)->left; + rightsub = (*pCurrentNode)->right; + if (leftsub == NULL) + { + free(*pCurrentNode); + *pCurrentNode = rightsub; + } + else if (rightsub == NULL) + { + free(*pCurrentNode); + *pCurrentNode = leftsub; + } + else + { + avl_tree_delete_loop(tree, &((*pCurrentNode)->left), \ + target_data, shorter, *pCurrentNode); + + if (*shorter != 0) + { + switch ((*pCurrentNode)->balance) + { + case -1: + (*pCurrentNode)->balance = 0; + break; + case 0: + (*pCurrentNode)->balance = 1; + *shorter = 0; + break; + case 1: + avlRightBalanceWhenDelete(pCurrentNode, shorter); + break; + } + } + return 1; + } + + *shorter = 1; + return 1; + } +} + +int avl_tree_delete(AVLTreeInfo *tree, void *data) +{ + int shorter; + + if (tree->root == NULL) + { + return 0; + } + + shorter = 0; + return avl_tree_delete_loop(tree, &(tree->root), data, &shorter, NULL); +} + +static int avl_tree_walk_loop(DataOpFunc data_op_func, \ + AVLTreeNode *pCurrentNode, void *args) +{ + int result; + + if (pCurrentNode->left != NULL) + { + result = avl_tree_walk_loop(data_op_func, \ + pCurrentNode->left, args); + if (result != 0) + { + return result; + } + } + + if ((result=data_op_func(pCurrentNode->data, args)) != 0) + { + return result; + } + + /* + if (pCurrentNode->balance >= -1 && pCurrentNode->balance <= 1) + { + //printf("==%d\n", pCurrentNode->balance); + } + else + { + printf("==bad %d!!!!!!!!!!!!\n", pCurrentNode->balance); + } + */ + + if (pCurrentNode->right != NULL) + { + result = avl_tree_walk_loop(data_op_func, \ + pCurrentNode->right, args); + } + + return result; +} + +int avl_tree_walk(AVLTreeInfo *tree, DataOpFunc data_op_func, void *args) +{ + if (tree->root == NULL) + { + return 0; + } + else + { + return avl_tree_walk_loop(data_op_func, tree->root, args); + } +} + +static void avl_tree_count_loop(AVLTreeNode *pCurrentNode, int *count) +{ + if (pCurrentNode->left != NULL) + { + avl_tree_count_loop(pCurrentNode->left, count); + } + + (*count)++; + + if (pCurrentNode->right != NULL) + { + avl_tree_count_loop(pCurrentNode->right, count); + } +} + +int avl_tree_count(AVLTreeInfo *tree) +{ + int count; + if (tree->root == NULL) + { + return 0; + } + + count = 0; + avl_tree_count_loop(tree->root, &count); + return count; +} + +int avl_tree_depth(AVLTreeInfo *tree) +{ + int depth; + AVLTreeNode *pNode; + + if (tree->root == NULL) + { + return 0; + } + + depth = 0; + pNode = tree->root; + while (pNode != NULL) + { + if (pNode->balance == -1) + { + pNode = pNode->left; + } + else + { + pNode = pNode->right; + } + depth++; + } + + return depth; +} + +/* +static void avl_tree_print_loop(AVLTreeNode *pCurrentNode) +{ + printf("%ld left: %ld, right: %ld, balance: %d\n", pCurrentNode->data, + pCurrentNode->left != NULL ? pCurrentNode->left->data : 0, + pCurrentNode->right != NULL ? pCurrentNode->right->data : 0, + pCurrentNode->balance); + + if (pCurrentNode->left != NULL) + { + avl_tree_print_loop(pCurrentNode->left); + } + + if (pCurrentNode->right != NULL) + { + avl_tree_print_loop(pCurrentNode->right); + } +} + +void avl_tree_print(AVLTreeInfo *tree) +{ + if (tree->root == NULL) + { + return; + } + + avl_tree_print_loop(tree->root); +} +*/ + diff --git a/common/avl_tree.h b/common/avl_tree.h new file mode 100644 index 0000000..1a9db45 --- /dev/null +++ b/common/avl_tree.h @@ -0,0 +1,47 @@ + +#ifndef AVL_TREE_H +#define AVL_TREE_H + +#include +#include +#include +#include "common_define.h" + +typedef struct tagAVLTreeNode { + void *data; + struct tagAVLTreeNode *left; + struct tagAVLTreeNode *right; + byte balance; +} AVLTreeNode; + +typedef int (*DataOpFunc) (void *data, void *args); + +typedef struct tagAVLTreeInfo { + AVLTreeNode *root; + FreeDataFunc free_data_func; + CompareFunc compare_func; +} AVLTreeInfo; + +#ifdef __cplusplus +extern "C" { +#endif + +int avl_tree_init(AVLTreeInfo *tree, FreeDataFunc free_data_func, \ + CompareFunc compare_func); +void avl_tree_destroy(AVLTreeInfo *tree); + +int avl_tree_insert(AVLTreeInfo *tree, void *data); +int avl_tree_replace(AVLTreeInfo *tree, void *data); +int avl_tree_delete(AVLTreeInfo *tree, void *data); +void *avl_tree_find(AVLTreeInfo *tree, void *target_data); +void *avl_tree_find_ge(AVLTreeInfo *tree, void *target_data); +int avl_tree_walk(AVLTreeInfo *tree, DataOpFunc data_op_func, void *args); +int avl_tree_count(AVLTreeInfo *tree); +int avl_tree_depth(AVLTreeInfo *tree); +//void avl_tree_print(AVLTreeInfo *tree); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/common/base64.c b/common/base64.c new file mode 100644 index 0000000..1ff77bf --- /dev/null +++ b/common/base64.c @@ -0,0 +1,393 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "base64.h" + +/** +* Marker value for chars we just ignore, e.g. \n \r high ascii +*/ +#define BASE64_IGNORE -1 + +/** +* Marker for = trailing pad +*/ +#define BASE64_PAD -2 + +/** +* determines how long the lines are that are generated by encode. +* Ignored by decode. +* @param length 0 means no newlines inserted. Must be a multiple of 4. +*/ +void base64_set_line_length(struct base64_context *context, const int length) +{ + context->line_length = (length / 4) * 4; +} + +/** + * How lines are separated. + * Ignored by decode. + * @param context->line_separator may be "" but not null. + * Usually contains only a combination of chars \n and \r. + * Could be any chars not in set A-Z a-z 0-9 + /. +*/ +void base64_set_line_separator(struct base64_context *context, \ + const char *pLineSeparator) +{ + context->line_sep_len = snprintf(context->line_separator, \ + sizeof(context->line_separator), "%s", pLineSeparator); +} + +void base64_init_ex(struct base64_context *context, const int nLineLength, \ + const unsigned char chPlus, const unsigned char chSplash, \ + const unsigned char chPad) +{ + int i; + + memset(context, 0, sizeof(struct base64_context)); + + context->line_length = nLineLength; + context->line_separator[0] = '\n'; + context->line_separator[1] = '\0'; + context->line_sep_len = 1; + + // build translate valueToChar table only once. + // 0..25 -> 'A'..'Z' + for (i=0; i<=25; i++) + { + context->valueToChar[i] = (char)('A'+i); + } + // 26..51 -> 'a'..'z' + for (i=0; i<=25; i++ ) + { + context->valueToChar[i+26] = (char)('a'+i); + } + // 52..61 -> '0'..'9' + for (i=0; i<=9; i++ ) + { + context->valueToChar[i+52] = (char)('0'+i); + } + context->valueToChar[62] = chPlus; + context->valueToChar[63] = chSplash; + + memset(context->charToValue, BASE64_IGNORE, sizeof(context->charToValue)); + for (i=0; i<64; i++ ) + { + context->charToValue[context->valueToChar[i]] = i; + } + + context->pad_ch = chPad; + context->charToValue[chPad] = BASE64_PAD; +} + +int base64_get_encode_length(struct base64_context *context, const int nSrcLen) +{ + // Each group or partial group of 3 bytes becomes four chars + // covered quotient + int outputLength; + + outputLength = ((nSrcLen + 2) / 3) * 4; + + // account for trailing newlines, on all but the very last line + if (context->line_length != 0) + { + int lines = (outputLength + context->line_length - 1) / + context->line_length - 1; + if ( lines > 0 ) + { + outputLength += lines * context->line_sep_len; + } + } + + return outputLength; +} + + /** + * Encode an arbitrary array of bytes as base64 printable ASCII. + * It will be broken into lines of 72 chars each. The last line is not + * terminated with a line separator. + * The output will always have an even multiple of data characters, + * exclusive of \n. It is padded out with =. + */ +char *base64_encode_ex(struct base64_context *context, const char *src, \ + const int nSrcLen, char *dest, int *dest_len, const bool bPad) +{ + int linePos; + int leftover; + int combined; + char *pDest; + int c0, c1, c2, c3; + unsigned char *pRaw; + unsigned char *pEnd; + const char *ppSrcs[2]; + int lens[2]; + char szPad[3]; + int k; + int loop; + + if (nSrcLen <= 0) + { + *dest = '\0'; + *dest_len = 0; + return dest; + } + + linePos = 0; + lens[0] = (nSrcLen / 3) * 3; + lens[1] = 3; + leftover = nSrcLen - lens[0]; + ppSrcs[0] = src; + ppSrcs[1] = szPad; + + szPad[0] = szPad[1] = szPad[2] = '\0'; + switch (leftover) + { + case 0: + default: + loop = 1; + break; + case 1: + loop = 2; + szPad[0] = src[nSrcLen-1]; + break; + case 2: + loop = 2; + szPad[0] = src[nSrcLen-2]; + szPad[1] = src[nSrcLen-1]; + break; + } + + pDest = dest; + for (k=0; k context->line_length) + { + if (context->line_length != 0) + { + memcpy(pDest, context->line_separator, context->line_sep_len); + pDest += context->line_sep_len; + } + linePos = 4; + } + + // get next three bytes in unsigned form lined up, + // in big-endian order + combined = ((*pRaw) << 16) | ((*(pRaw+1)) << 8) | (*(pRaw+2)); + + // break those 24 bits into a 4 groups of 6 bits, + // working LSB to MSB. + c3 = combined & 0x3f; + combined >>= 6; + c2 = combined & 0x3f; + combined >>= 6; + c1 = combined & 0x3f; + combined >>= 6; + c0 = combined & 0x3f; + + // Translate into the equivalent alpha character + // emitting them in big-endian order. + *pDest++ = context->valueToChar[c0]; + *pDest++ = context->valueToChar[c1]; + *pDest++ = context->valueToChar[c2]; + *pDest++ = context->valueToChar[c3]; + } + } + + *pDest = '\0'; + *dest_len = pDest - dest; + + // deal with leftover bytes + switch (leftover) + { + case 0: + default: + // nothing to do + break; + case 1: + // One leftover byte generates xx== + if (bPad) + { + *(pDest-1) = context->pad_ch; + *(pDest-2) = context->pad_ch; + } + else + { + *(pDest-2) = '\0'; + *dest_len -= 2; + } + break; + case 2: + // Two leftover bytes generates xxx= + if (bPad) + { + *(pDest-1) = context->pad_ch; + } + else + { + *(pDest-1) = '\0'; + *dest_len -= 1; + } + break; + } // end switch; + + return dest; +} + +char *base64_decode_auto(struct base64_context *context, const char *src, \ + const int nSrcLen, char *dest, int *dest_len) +{ + int nRemain; + int nPadLen; + int nNewLen; + char tmpBuff[256]; + char *pBuff; + + nRemain = nSrcLen % 4; + if (nRemain == 0) + { + return base64_decode(context, src, nSrcLen, dest, dest_len); + } + + nPadLen = 4 - nRemain; + nNewLen = nSrcLen + nPadLen; + if (nNewLen <= sizeof(tmpBuff)) + { + pBuff = tmpBuff; + } + else + { + pBuff = (char *)malloc(nNewLen); + if (pBuff == NULL) + { + fprintf(stderr, "Can't malloc %d bytes\n", \ + nSrcLen + nPadLen + 1); + *dest_len = 0; + *dest = '\0'; + return dest; + } + } + + memcpy(pBuff, src, nSrcLen); + memset(pBuff + nSrcLen, context->pad_ch, nPadLen); + + base64_decode(context, pBuff, nNewLen, dest, dest_len); + + if (pBuff != tmpBuff) + { + free(pBuff); + } + + return dest; +} + +/** +* decode a well-formed complete base64 string back into an array of bytes. +* It must have an even multiple of 4 data characters (not counting \n), +* padded out with = as needed. +*/ +char *base64_decode(struct base64_context *context, const char *src, \ + const int nSrcLen, char *dest, int *dest_len) +{ + // tracks where we are in a cycle of 4 input chars. + int cycle; + + // where we combine 4 groups of 6 bits and take apart as 3 groups of 8. + int combined; + + // will be an even multiple of 4 chars, plus some embedded \n + int dummies; + int value; + unsigned char *pSrc; + unsigned char *pSrcEnd; + char *pDest; + + cycle = 0; + combined = 0; + dummies = 0; + pDest = dest; + pSrcEnd = (unsigned char *)src + nSrcLen; + for (pSrc=(unsigned char *)src; pSrccharToValue[*pSrc]; + switch (value) + { + case BASE64_IGNORE: + // e.g. \n, just ignore it. + break; + case BASE64_PAD: + value = 0; + dummies++; + // fallthrough + default: + /* regular value character */ + switch (cycle) + { + case 0: + combined = value; + cycle = 1; + break; + case 1: + combined <<= 6; + combined |= value; + cycle = 2; + break; + case 2: + combined <<= 6; + combined |= value; + cycle = 3; + break; + case 3: + combined <<= 6; + combined |= value; + // we have just completed a cycle of 4 chars. + // the four 6-bit values are in combined in big-endian order + // peel them off 8 bits at a time working lsb to msb + // to get our original 3 8-bit bytes back + + *pDest++ = (char)(combined >> 16); + *pDest++ = (char)((combined & 0x0000FF00) >> 8); + *pDest++ = (char)(combined & 0x000000FF); + + cycle = 0; + break; + } + break; + } + } // end for + + if (cycle != 0) + { + *dest = '\0'; + *dest_len = 0; + fprintf(stderr, "Input to decode not an even multiple of " \ + "4 characters; pad with %c\n", context->pad_ch); + return dest; + } + + *dest_len = (pDest - dest) - dummies; + *(dest + (*dest_len)) = '\0'; + + return dest; +} + diff --git a/common/base64.h b/common/base64.h new file mode 100644 index 0000000..6c85007 --- /dev/null +++ b/common/base64.h @@ -0,0 +1,134 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//base64.h + +#ifndef _BASE64_H +#define _BASE64_H + +#include "common_define.h" + +struct base64_context +{ + char line_separator[16]; + int line_sep_len; + + /** + * max chars per line, excluding line_separator. A multiple of 4. + */ + int line_length; + + /** + * letter of the alphabet used to encode binary values 0..63 + */ + unsigned char valueToChar[64]; + + /** + * binary value encoded by a given letter of the alphabet 0..63 + */ + int charToValue[256]; + int pad_ch; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** use stardand base64 charset + */ +#define base64_init(context, nLineLength) \ + base64_init_ex(context, nLineLength, '+', '/', '=') + + +/** stardand base64 encode + */ +#define base64_encode(context, src, nSrcLen, dest, dest_len) \ + base64_encode_ex(context, src, nSrcLen, dest, dest_len, true) + +/** base64 init function, before use base64 function, you should call + * this function + * parameters: + * context: the base64 context + * nLineLength: length of a line, 0 for never add line seperator + * chPlus: specify a char for base64 char plus (+) + * chSplash: specify a char for base64 char splash (/) + * chPad: specify a char for base64 padding char = + * return: none + */ +void base64_init_ex(struct base64_context *context, const int nLineLength, \ + const unsigned char chPlus, const unsigned char chSplash, \ + const unsigned char chPad); + +/** calculate the encoded length of the source buffer + * parameters: + * context: the base64 context + * nSrcLen: the source buffer length + * return: the encoded length of the source buffer + */ +int base64_get_encode_length(struct base64_context *context, const int nSrcLen); + +/** base64 encode buffer + * parameters: + * context: the base64 context + * src: the source buffer + * nSrcLen: the source buffer length + * dest: the dest buffer + * dest_len: return dest buffer length + * bPad: if padding + * return: the encoded buffer + */ +char *base64_encode_ex(struct base64_context *context, const char *src, \ + const int nSrcLen, char *dest, int *dest_len, const bool bPad); + +/** base64 decode buffer, work only with padding source string + * parameters: + * context: the base64 context + * src: the source buffer with padding + * nSrcLen: the source buffer length + * dest: the dest buffer + * dest_len: return dest buffer length + * return: the decoded buffer + */ +char *base64_decode(struct base64_context *context, const char *src, \ + const int nSrcLen, char *dest, int *dest_len); + +/** base64 decode buffer, can work with no padding source string + * parameters: + * context: the base64 context + * src: the source buffer with padding or no padding + * nSrcLen: the source buffer length + * dest: the dest buffer + * dest_len: return dest buffer length + * return: the decoded buffer + */ +char *base64_decode_auto(struct base64_context *context, const char *src, \ + const int nSrcLen, char *dest, int *dest_len); + +/** set line separator string, such as \n or \r\n + * parameters: + * context: the base64 context + * pLineSeparator: the line separator string + * return: none + */ +void base64_set_line_separator(struct base64_context *context, \ + const char *pLineSeparator); + +/** set line length, 0 for never add line separators + * parameters: + * context: the base64 context + * length: the line length + * return: none + */ +void base64_set_line_length(struct base64_context *context, const int length); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/chain.c b/common/chain.c new file mode 100644 index 0000000..a23e5ab --- /dev/null +++ b/common/chain.c @@ -0,0 +1,325 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include "chain.h" +//#include "use_mmalloc.h" + +void chain_init(ChainList *pList, const int type, FreeDataFunc freeDataFunc, \ + CompareFunc compareFunc) +{ + if (pList == NULL) + { + return; + } + + pList->head = NULL; + pList->tail = NULL; + pList->type = type; + pList->freeDataFunc = freeDataFunc; + pList->compareFunc = compareFunc; + + return; +} + +void chain_destroy(ChainList *pList) +{ + ChainNode *pNode; + ChainNode *pDeleted; + if (pList == NULL || pList->head == NULL) + { + return; + } + + pNode = pList->head; + while (pNode != NULL) + { + pDeleted = pNode; + pNode = pNode->next; + + freeChainNode(pList, pDeleted); + } + + pList->head = pList->tail = NULL; +} + +void freeChainNode(ChainList *pList, ChainNode *pChainNode) +{ + if (pList->freeDataFunc != NULL) + { + pList->freeDataFunc(pChainNode->data); + } + + free(pChainNode); +} + +int insertNodePrior(ChainList *pList, void *data) +{ + ChainNode *pNode; + if (pList == NULL) + { + return EINVAL; + } + + pNode = (ChainNode *)malloc(sizeof(ChainNode)); + if (pNode == NULL) + { + return ENOMEM; + } + + pNode->data = data; + pNode->next = pList->head; + + pList->head = pNode; + if (pList->tail == NULL) + { + pList->tail = pNode; + } + + return 0; +} + +int appendNode(ChainList *pList, void *data) +{ + ChainNode *pNode; + if (pList == NULL) + { + return EINVAL; + } + + pNode = (ChainNode *)malloc(sizeof(ChainNode)); + if (pNode == NULL) + { + return ENOMEM; + } + + pNode->data = data; + pNode->next = NULL; + + if (pList->tail != NULL) + { + pList->tail->next = pNode; + } + + pList->tail = pNode; + if (pList->head == NULL) + { + pList->head = pNode; + } + + return 0; +} + +int insertNodeAsc(ChainList *pList, void *data) +{ + ChainNode *pNew; + ChainNode *pNode; + ChainNode *pPrevious; + if (pList == NULL || pList->compareFunc == NULL) + { + return EINVAL; + } + + pNew = (ChainNode *)malloc(sizeof(ChainNode)); + if (pNew == NULL) + { + return ENOMEM; + } + + pNew->data = data; + + pPrevious = NULL; + pNode = pList->head; + while (pNode != NULL && pList->compareFunc(pNode->data, data) < 0) + { + pPrevious = pNode; + pNode = pNode->next; + } + + pNew->next = pNode; + if (pPrevious == NULL) + { + pList->head = pNew; + } + else + { + pPrevious->next = pNew; + } + + if (pNode == NULL) + { + pList->tail = pNew; + } + + return 0; +} + +int addNode(ChainList *pList, void *data) +{ + if (pList->type == CHAIN_TYPE_INSERT) + { + return insertNodePrior(pList, data); + } + else if (pList->type == CHAIN_TYPE_APPEND) + { + return appendNode(pList, data); + } + else + { + return insertNodeAsc(pList, data); + } +} + +void deleteNodeEx(ChainList *pList, ChainNode *pPreviousNode, \ + ChainNode *pDeletedNode) +{ + if (pDeletedNode == pList->head) + { + pList->head = pDeletedNode->next; + } + else + { + pPreviousNode->next = pDeletedNode->next; + } + + if (pDeletedNode == pList->tail) + { + pList->tail = pPreviousNode; + } + + freeChainNode(pList, pDeletedNode); +} + +void deleteToNodePrevious(ChainList *pList, ChainNode *pPreviousNode, \ + ChainNode *pDeletedNext) +{ + ChainNode *pNode; + ChainNode *pDeletedNode; + + if (pPreviousNode == NULL) + { + pNode = pList->head; + pList->head = pDeletedNext; + } + else + { + pNode = pPreviousNode->next; + pPreviousNode->next = pDeletedNext; + } + + while (pNode != NULL && pNode != pDeletedNext) + { + pDeletedNode = pNode; + pNode = pNode->next; + freeChainNode(pList, pDeletedNode); + } + + if (pDeletedNext == NULL) + { + pList->tail = pPreviousNode; + } +} + +void *chain_pop_head(ChainList *pList) +{ + ChainNode *pDeletedNode; + void *data; + + pDeletedNode = pList->head; + if (pDeletedNode == NULL) + { + return NULL; + } + + pList->head = pDeletedNode->next; + if (pList->head == NULL) + { + pList->tail = NULL; + } + + data = pDeletedNode->data; + free(pDeletedNode); + + return data; +} + +int chain_count(ChainList *pList) +{ + ChainNode *pNode; + int count; + + count = 0; + pNode = pList->head; + while (pNode != NULL) + { + count++; + pNode = pNode->next; + } + + return count; +} + +int deleteNode(ChainList *pList, void *data, bool bDeleteAll) +{ + ChainNode *pNode; + ChainNode *pPrevious; + ChainNode *pDeleted; + int nCount; + int nCompareRes; + + if (pList == NULL || pList->compareFunc == NULL) + { + return -EINVAL; + } + + nCount = 0; + pPrevious = NULL; + pNode = pList->head; + while (pNode != NULL) + { + nCompareRes = pList->compareFunc(pNode->data, data); + if (nCompareRes == 0) + { + pDeleted = pNode; + pNode = pNode->next; + + deleteNodeEx(pList, pPrevious, pDeleted); + nCount++; + + if (!bDeleteAll) + { + break; + } + + continue; + } + else if(nCompareRes > 0 && pList->type == CHAIN_TYPE_SORTED) + { + break; + } + + pPrevious = pNode; + pNode = pNode->next; + } + + return nCount; +} + +int deleteOne(ChainList *pList, void *data) +{ + return deleteNode(pList, data, false); +} + +int deleteAll(ChainList *pList, void *data) +{ + return deleteNode(pList, data, true); +} + diff --git a/common/chain.h b/common/chain.h new file mode 100644 index 0000000..9d18a3d --- /dev/null +++ b/common/chain.h @@ -0,0 +1,148 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef CHAIN_H +#define CHAIN_H + +#include "common_define.h" + +#define CHAIN_TYPE_INSERT 0 //insert new node before head +#define CHAIN_TYPE_APPEND 1 //insert new node after tail +#define CHAIN_TYPE_SORTED 2 //sorted chain + +typedef struct tagChainNode +{ + void *data; + struct tagChainNode *next; +} ChainNode; + +typedef struct +{ + int type; + ChainNode *head; + ChainNode *tail; + FreeDataFunc freeDataFunc; + CompareFunc compareFunc; +} ChainList; + +#ifdef __cplusplus +extern "C" { +#endif + +/** chain init function + * parameters: + * pList: the chain list + * type: chain type, value is one of following: + * CHAIN_TYPE_INSERT: insert new node before head + * CHAIN_TYPE_APPEND: insert new node after tail + * CHAIN_TYPE_SORTED: sorted chain + * freeDataFunc: free data function pointer, can be NULL + * compareFunc: compare data function pointer, can be NULL, + * must set when type is CHAIN_TYPE_SORTED + * return: none + */ +void chain_init(ChainList *pList, const int type, FreeDataFunc freeDataFunc, \ + CompareFunc compareFunc); + +/** destroy chain + * parameters: + * pList: the chain list + * return: none + */ +void chain_destroy(ChainList *pList); + + +/** get the chain node count + * parameters: + * pList: the chain list + * return: chain node count + */ +int chain_count(ChainList *pList); + +/** add a new node to the chain + * parameters: + * pList: the chain list + * data: the data to add + * return: error no, 0 for success, != 0 fail + */ +int addNode(ChainList *pList, void *data); + +/** free the chain node + * parameters: + * pList: the chain list + * pChainNode: the chain node to free + * return: none + */ +void freeChainNode(ChainList *pList, ChainNode *pChainNode); + +/** delete the chain node + * parameters: + * pList: the chain list + * pPreviousNode: the previous chain node + * pDeletedNode: the chain node to delete + * return: none + */ +void deleteNodeEx(ChainList *pList, ChainNode *pPreviousNode, \ + ChainNode *pDeletedNode); + +/** delete the chain nodes from pPreviousNode->next to pDeletedNext + * (not including pDeletedNext) + * parameters: + * pList: the chain list + * pPreviousNode: the previous chain node, delete from pPreviousNode->next + * pDeletedNext: the chain node after the deleted node + * return: none + */ +void deleteToNodePrevious(ChainList *pList, ChainNode *pPreviousNode, \ + ChainNode *pDeletedNext); + +/** delete the chain node using data compare function + * parameters: + * pList: the chain list + * data: the first node whose data equals this will be deleted + * return: delete chain node count, < 0 fail + */ +int deleteOne(ChainList *pList, void *data); + +/** delete all chain nodes using data compare function + * parameters: + * pList: the chain list + * data: the node whose data equals this will be deleted + * return: delete chain node count, < 0 fail + */ +int deleteAll(ChainList *pList, void *data); + +/** pop up a chain nodes from chain head + * parameters: + * pList: the chain list + * return: the head node, return NULL when the chain is empty + */ +void *chain_pop_head(ChainList *pList); + +/** add a chain nodes before the chain head + * parameters: + * pList: the chain list + * data: the node to insert + * return: error no, 0 for success, != 0 fail + */ +int insertNodePrior(ChainList *pList, void *data); + +/** add a chain nodes after the chain tail + * parameters: + * pList: the chain list + * data: the node to insert + * return: error no, 0 for success, != 0 fail + */ +int appendNode(ChainList *pList, void *data); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/common_define.h b/common/common_define.h new file mode 100644 index 0000000..5cb8128 --- /dev/null +++ b/common/common_define.h @@ -0,0 +1,169 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//common_define.h + +#ifndef _COMMON_DEFINE_H_ +#define _COMMON_DEFINE_H_ + +#include +#include + +#ifdef WIN32 + +#include +#include +typedef UINT in_addr_t; +#define FILE_SEPERATOR "\\" +#define THREAD_ENTRANCE_FUNC_DECLARE DWORD WINAPI +#define THREAD_RETURN_VALUE 0 +typedef DWORD (WINAPI *ThreadEntranceFunc)(LPVOID lpThreadParameter); +#else + +#include +#include +#include +#include +#include +#include +#define FILE_SEPERATOR "/" +typedef int SOCKET; +#define closesocket close +#define INVALID_SOCKET -1 +#define THREAD_ENTRANCE_FUNC_DECLARE void * +typedef void *LPVOID; +#define THREAD_RETURN_VALUE NULL +typedef void * (*ThreadEntranceFunc)(LPVOID lpThreadParameter); + +#endif + +#ifndef WIN32 +extern int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int kind); +#endif + +#ifdef OS_LINUX +#define PTHREAD_MUTEX_ERRORCHECK PTHREAD_MUTEX_ERRORCHECK_NP +#endif + +#include "_os_bits.h" + +#ifdef OS_BITS + #if OS_BITS == 64 + #define INT64_PRINTF_FORMAT "%ld" + #else + #define INT64_PRINTF_FORMAT "%lld" + #endif +#else + #define INT64_PRINTF_FORMAT "%lld" +#endif + +#ifdef OFF_BITS + #if OFF_BITS == 64 + #define OFF_PRINTF_FORMAT INT64_PRINTF_FORMAT + #else + #define OFF_PRINTF_FORMAT "%d" + #endif +#else + #define OFF_PRINTF_FORMAT INT64_PRINTF_FORMAT +#endif + +#ifndef WIN32 +#define USE_SENDFILE +#endif + +#define MAX_PATH_SIZE 256 +#define LOG_FILE_DIR "logs" +#define CONF_FILE_DIR "conf" +#define DEFAULT_CONNECT_TIMEOUT 30 +#define DEFAULT_NETWORK_TIMEOUT 30 +#define DEFAULT_MAX_CONNECTONS 256 +#define DEFAULT_WORK_THREADS 4 +#define SYNC_LOG_BUFF_DEF_INTERVAL 10 +#define TIME_NONE -1 + +#define IP_ADDRESS_SIZE 16 +#define INFINITE_FILE_SIZE (256 * 1024LL * 1024 * 1024 * 1024 * 1024LL) + +#ifndef __cplusplus +#ifndef true +typedef char bool; +#define true 1 +#define false 0 +#endif +#endif + +#ifndef byte +#define byte signed char +#endif + +#ifndef ubyte +#define ubyte unsigned char +#endif + +#ifndef WIN32 +#ifndef INADDR_NONE +#define INADDR_NONE ((in_addr_t) 0xffffffff) +#endif +#endif + +#ifndef ECANCELED +#define ECANCELED 125 +#endif + +#ifndef ENONET +#define ENONET 64 /* Machine is not on the network */ +#endif + +#define IS_UPPER_HEX(ch) ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F')) +#define STRERROR(no) (strerror(no) != NULL ? strerror(no) : "Unkown error") + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct +{ + byte hour; + byte minute; +} TimeInfo; + +typedef struct +{ + char major; + char minor; +} Version; + +typedef struct +{ + char *key; + char *value; +} KeyValuePair; + +typedef struct +{ + char *buff; + int alloc_size; + int length; +} BufferInfo; + +typedef void (*FreeDataFunc)(void *ptr); +typedef int (*CompareFunc)(void *p1, void *p2); +typedef void* (*MallocFunc)(size_t size); + +#define TO_UPPERCASE(c) (((c) >= 'a' && (c) <= 'z') ? (c) - 32 : c) +#define MEM_ALIGN(x) (((x) + 7) & (~7)) + +#ifdef WIN32 +#define strcasecmp _stricmp +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/common/connection_pool.c b/common/connection_pool.c new file mode 100644 index 0000000..e272862 --- /dev/null +++ b/common/connection_pool.c @@ -0,0 +1,346 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include "logger.h" +#include "sockopt.h" +#include "shared_func.h" +#include "sched_thread.h" +#include "connection_pool.h" + +int conn_pool_init(ConnectionPool *cp, int connect_timeout, \ + const int max_count_per_entry, const int max_idle_time) +{ + int result; + + if ((result=init_pthread_lock(&cp->lock)) != 0) + { + return result; + } + cp->connect_timeout = connect_timeout; + cp->max_count_per_entry = max_count_per_entry; + cp->max_idle_time = max_idle_time; + + return hash_init(&(cp->hash_array), simple_hash, 1024, 0.75); +} + +void conn_pool_destroy(ConnectionPool *cp) +{ + pthread_mutex_lock(&cp->lock); + hash_destroy(&(cp->hash_array)); + pthread_mutex_unlock(&cp->lock); + + pthread_mutex_destroy(&cp->lock); +} + +void conn_pool_disconnect_server(ConnectionInfo *pConnection) +{ + if (pConnection->sock >= 0) + { + close(pConnection->sock); + pConnection->sock = -1; + } +} + +int conn_pool_connect_server(ConnectionInfo *pConnection, \ + const int connect_timeout) +{ + int result; + + if (pConnection->sock >= 0) + { + close(pConnection->sock); + } + + pConnection->sock = socket(AF_INET, SOCK_STREAM, 0); + if(pConnection->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s", __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + if ((result=tcpsetnonblockopt(pConnection->sock)) != 0) + { + close(pConnection->sock); + pConnection->sock = -1; + return result; + } + + if ((result=connectserverbyip_nb(pConnection->sock, \ + pConnection->ip_addr, pConnection->port, \ + connect_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to %s:%d fail, errno: %d, " \ + "error info: %s", __LINE__, pConnection->ip_addr, \ + pConnection->port, result, STRERROR(result)); + + close(pConnection->sock); + pConnection->sock = -1; + return result; + } + + return 0; +} + +static int conn_pool_get_key(const ConnectionInfo *conn, char *key, int *key_len) +{ + struct in_addr sin_addr; + + if (inet_aton(conn->ip_addr, &sin_addr) == 0) + { + *key_len = 0; + return EINVAL; + } + + int2buff(sin_addr.s_addr, key); + *key_len = 4 + sprintf(key + 4, "%d", conn->port); + + return 0; +} + +ConnectionInfo *conn_pool_get_connection(ConnectionPool *cp, + const ConnectionInfo *conn, int *err_no) +{ + char key[32]; + int key_len; + int bytes; + char *p; + ConnectionManager *cm; + ConnectionNode *node; + ConnectionInfo *ci; + time_t current_time; + + *err_no = conn_pool_get_key(conn, key, &key_len); + if (*err_no != 0) + { + return NULL; + } + + pthread_mutex_lock(&cp->lock); + cm = (ConnectionManager *)hash_find(&cp->hash_array, key, key_len); + if (cm == NULL) + { + cm = (ConnectionManager *)malloc(sizeof(ConnectionManager)); + if (cm == NULL) + { + *err_no = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, " \ + "error info: %s", __LINE__, \ + (int)sizeof(ConnectionManager), \ + *err_no, STRERROR(*err_no)); + pthread_mutex_unlock(&cp->lock); + return NULL; + } + + cm->head = NULL; + cm->total_count = 0; + cm->free_count = 0; + if ((*err_no=init_pthread_lock(&cm->lock)) != 0) + { + pthread_mutex_unlock(&cp->lock); + return NULL; + } + hash_insert(&cp->hash_array, key, key_len, cm); + } + pthread_mutex_unlock(&cp->lock); + + current_time = get_current_time(); + pthread_mutex_lock(&cm->lock); + while (1) + { + if (cm->head == NULL) + { + if ((cp->max_count_per_entry > 0) && + (cm->total_count >= cp->max_count_per_entry)) + { + *err_no = ENOSPC; + logError("file: "__FILE__", line: %d, " \ + "connections: %d of server %s:%d " \ + "exceed limit: %d", __LINE__, \ + cm->total_count, conn->ip_addr, \ + conn->port, cp->max_count_per_entry); + pthread_mutex_unlock(&cm->lock); + return NULL; + } + + bytes = sizeof(ConnectionInfo) + sizeof(ConnectionNode); + p = (char *)malloc(bytes); + if (p == NULL) + { + *err_no = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, " \ + "error info: %s", __LINE__, \ + bytes, *err_no, STRERROR(*err_no)); + pthread_mutex_unlock(&cm->lock); + return NULL; + } + + node = (ConnectionNode *)(p + sizeof(ConnectionInfo)); + node->conn = (ConnectionInfo *)p; + node->manager = cm; + node->next = NULL; + node->atime = 0; + + cm->total_count++; + pthread_mutex_unlock(&cm->lock); + + memcpy(node->conn, conn, sizeof(ConnectionInfo)); + node->conn->sock = -1; + *err_no = conn_pool_connect_server(node->conn, \ + cp->connect_timeout); + if (*err_no != 0) + { + free(p); + return NULL; + } + + logDebug("file: "__FILE__", line: %d, " \ + "server %s:%d, new connection: %d, " \ + "total_count: %d, free_count: %d", \ + __LINE__, conn->ip_addr, conn->port, \ + node->conn->sock, cm->total_count, \ + cm->free_count); + return node->conn; + } + else + { + node = cm->head; + ci = node->conn; + cm->head = node->next; + cm->free_count--; + + if (current_time - node->atime > cp->max_idle_time) + { + cm->total_count--; + + logDebug("file: "__FILE__", line: %d, " \ + "server %s:%d, connection: %d idle " \ + "time: %d exceeds max idle time: %d, "\ + "total_count: %d, free_count: %d", \ + __LINE__, conn->ip_addr, conn->port, \ + ci->sock, \ + (int)(current_time - node->atime), \ + cp->max_idle_time, cm->total_count, \ + cm->free_count); + + conn_pool_disconnect_server(ci); + free(ci); + continue; + } + + pthread_mutex_unlock(&cm->lock); + logDebug("file: "__FILE__", line: %d, " \ + "server %s:%d, reuse connection: %d, " \ + "total_count: %d, free_count: %d", + __LINE__, conn->ip_addr, conn->port, + ci->sock, cm->total_count, cm->free_count); + return ci; + } + } +} + +int conn_pool_close_connection_ex(ConnectionPool *cp, ConnectionInfo *conn, + const bool bForce) +{ + char key[32]; + int result; + int key_len; + ConnectionManager *cm; + ConnectionNode *node; + + result = conn_pool_get_key(conn, key, &key_len); + if (result != 0) + { + return result; + } + + pthread_mutex_lock(&cp->lock); + cm = (ConnectionManager *)hash_find(&cp->hash_array, key, key_len); + pthread_mutex_unlock(&cp->lock); + if (cm == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "hash entry of server %s:%d not exist", __LINE__, \ + conn->ip_addr, conn->port); + return ENOENT; + } + + node = (ConnectionNode *)(((char *)conn) + sizeof(ConnectionInfo)); + if (node->manager != cm) + { + logError("file: "__FILE__", line: %d, " \ + "manager of server entry %s:%d is invalid!", \ + __LINE__, conn->ip_addr, conn->port); + return EINVAL; + } + + pthread_mutex_lock(&cm->lock); + if (bForce) + { + cm->total_count--; + + logDebug("file: "__FILE__", line: %d, " \ + "server %s:%d, release connection: %d, " \ + "total_count: %d, free_count: %d", + __LINE__, conn->ip_addr, conn->port, + conn->sock, cm->total_count, cm->free_count); + + conn_pool_disconnect_server(conn); + free(conn); + } + else + { + node->atime = get_current_time(); + node->next = cm->head; + cm->head = node; + cm->free_count++; + + logDebug("file: "__FILE__", line: %d, " \ + "server %s:%d, free connection: %d, " \ + "total_count: %d, free_count: %d", + __LINE__, conn->ip_addr, conn->port, + conn->sock, cm->total_count, cm->free_count); + } + pthread_mutex_unlock(&cm->lock); + + return 0; +} + +static int _conn_count_walk(const int index, const HashData *data, void *args) +{ + int *count; + ConnectionManager *cm; + ConnectionNode *node; + + count = (int *)args; + cm = (ConnectionManager *)data->value; + node = cm->head; + while (node != NULL) + { + (*count)++; + node = node->next; + } + + return 0; +} + +int conn_pool_get_connection_count(ConnectionPool *cp) +{ + int count; + count = 0; + hash_walk(&cp->hash_array, _conn_count_walk, &count); + return count; +} + diff --git a/common/connection_pool.h b/common/connection_pool.h new file mode 100644 index 0000000..265693f --- /dev/null +++ b/common/connection_pool.h @@ -0,0 +1,86 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//connection_pool.h + +#ifndef _CONNECTION_POOL_H +#define _CONNECTION_POOL_H + +#include +#include +#include +#include +#include "common_define.h" +#include "pthread_func.h" +#include "hash.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct +{ + int sock; + int port; + char ip_addr[IP_ADDRESS_SIZE]; +} ConnectionInfo; + +struct tagConnectionManager; + +typedef struct tagConnectionNode { + ConnectionInfo *conn; + struct tagConnectionManager *manager; + struct tagConnectionNode *next; + time_t atime; //last access time +} ConnectionNode; + +typedef struct tagConnectionManager { + ConnectionNode *head; + int total_count; //total connections + int free_count; //free connections + pthread_mutex_t lock; +} ConnectionManager; + +typedef struct tagConnectionPool { + HashArray hash_array; //key is ip:port, value is ConnectionManager + pthread_mutex_t lock; + int connect_timeout; + int max_count_per_entry; //0 means no limit + + /* + connections whose the idle time exceeds this time will be closed + */ + int max_idle_time; +} ConnectionPool; + +int conn_pool_init(ConnectionPool *cp, int connect_timeout, \ + const int max_count_per_entry, const int max_idle_time); +void conn_pool_destroy(ConnectionPool *cp); + +ConnectionInfo *conn_pool_get_connection(ConnectionPool *cp, + const ConnectionInfo *conn, int *err_no); + +#define conn_pool_close_connection(cp, conn) \ + conn_pool_close_connection_ex(cp, conn, false) + +int conn_pool_close_connection_ex(ConnectionPool *cp, ConnectionInfo *conn, + const bool bForce); + +void conn_pool_disconnect_server(ConnectionInfo *pConnection); + +int conn_pool_connect_server(ConnectionInfo *pConnection, \ + const int connect_timeout); + +int conn_pool_get_connection_count(ConnectionPool *cp); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/fast_mblock.c b/common/fast_mblock.c new file mode 100644 index 0000000..488a084 --- /dev/null +++ b/common/fast_mblock.c @@ -0,0 +1,225 @@ +//fast_mblock.c + +#include +#include +#include +#include "fast_mblock.h" +#include "logger.h" +#include "shared_func.h" +#include "pthread_func.h" + +int fast_mblock_init(struct fast_mblock_man *mblock, const int element_size, \ + const int alloc_elements_once) +{ + int result; + + if (element_size <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid block size: %d", \ + __LINE__, element_size); + return EINVAL; + } + + mblock->element_size = element_size; + if (alloc_elements_once > 0) + { + mblock->alloc_elements_once = alloc_elements_once; + } + else + { + int block_size; + block_size = MEM_ALIGN(sizeof(struct fast_mblock_node) \ + + element_size); + mblock->alloc_elements_once = (1024 * 1024) / block_size; + } + + if ((result=init_pthread_lock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_lock fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + mblock->malloc_chain_head = NULL; + mblock->free_chain_head = NULL; + + return 0; +} + +static int fast_mblock_prealloc(struct fast_mblock_man *mblock) +{ + struct fast_mblock_node *pNode; + struct fast_mblock_malloc *pMallocNode; + char *pNew; + char *pTrunkStart; + char *p; + char *pLast; + int block_size; + int alloc_size; + + block_size = MEM_ALIGN(sizeof(struct fast_mblock_node) + \ + mblock->element_size); + alloc_size = sizeof(struct fast_mblock_malloc) + block_size * \ + mblock->alloc_elements_once; + + pNew = (char *)malloc(alloc_size); + if (pNew == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, alloc_size, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(pNew, 0, alloc_size); + + pMallocNode = (struct fast_mblock_malloc *)pNew; + + pTrunkStart = pNew + sizeof(struct fast_mblock_malloc); + pLast = pNew + (alloc_size - block_size); + for (p=pTrunkStart; pnext = (struct fast_mblock_node *)(p + block_size); + } + + ((struct fast_mblock_node *)pLast)->next = NULL; + mblock->free_chain_head = (struct fast_mblock_node *)pTrunkStart; + + pMallocNode->next = mblock->malloc_chain_head; + mblock->malloc_chain_head = pMallocNode; + + return 0; +} + +void fast_mblock_destroy(struct fast_mblock_man *mblock) +{ + struct fast_mblock_malloc *pMallocNode; + struct fast_mblock_malloc *pMallocTmp; + + if (mblock->malloc_chain_head == NULL) + { + return; + } + + pMallocNode = mblock->malloc_chain_head; + while (pMallocNode != NULL) + { + pMallocTmp = pMallocNode; + pMallocNode = pMallocNode->next; + + free(pMallocTmp); + } + mblock->malloc_chain_head = NULL; + mblock->free_chain_head = NULL; + + pthread_mutex_destroy(&(mblock->lock)); +} + +struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock) +{ + struct fast_mblock_node *pNode; + int result; + + if ((result=pthread_mutex_lock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return NULL; + } + + if (mblock->free_chain_head != NULL) + { + pNode = mblock->free_chain_head; + mblock->free_chain_head = pNode->next; + } + else + { + if ((result=fast_mblock_prealloc(mblock)) == 0) + { + pNode = mblock->free_chain_head; + mblock->free_chain_head = pNode->next; + } + else + { + pNode = NULL; + } + } + + if ((result=pthread_mutex_unlock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return pNode; +} + +int fast_mblock_free(struct fast_mblock_man *mblock, \ + struct fast_mblock_node *pNode) +{ + int result; + + if ((result=pthread_mutex_lock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + pNode->next = mblock->free_chain_head; + mblock->free_chain_head = pNode; + + if ((result=pthread_mutex_unlock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return 0; +} + +int fast_mblock_count(struct fast_mblock_man *mblock) +{ + struct fast_mblock_node *pNode; + int count; + int result; + + if ((result=pthread_mutex_lock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return -1; + } + + count = 0; + pNode = mblock->free_chain_head; + while (pNode != NULL) + { + pNode = pNode->next; + count++; + } + + if ((result=pthread_mutex_unlock(&(mblock->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return count; +} + diff --git a/common/fast_mblock.h b/common/fast_mblock.h new file mode 100644 index 0000000..496c9c7 --- /dev/null +++ b/common/fast_mblock.h @@ -0,0 +1,100 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fast_mblock.h + +#ifndef _FAST_MBLOCK_H +#define _FAST_MBLOCK_H + +#include +#include +#include +#include +#include "common_define.h" +#include "chain.h" + +/* free node chain */ +struct fast_mblock_node +{ + struct fast_mblock_node *next; + char data[0]; //the data buffer +}; + +/* malloc chain */ +struct fast_mblock_malloc +{ + struct fast_mblock_malloc *next; +}; + +struct fast_mblock_man +{ + struct fast_mblock_node *free_chain_head; //free node chain + struct fast_mblock_malloc *malloc_chain_head; //malloc chain to be freed + int element_size; //element size + int alloc_elements_once; //alloc elements once + pthread_mutex_t lock; //the lock for read / write free node chain +}; + +#define fast_mblock_to_node_ptr(data_ptr) \ + (struct fast_mblock_node *)(data_ptr - ((size_t)(char *) \ + &((struct fast_mblock_node *)0)->data)) + +#ifdef __cplusplus +extern "C" { +#endif + +/** +mblock init +parameters: + mblock: the mblock pointer + element_size: element size, such as sizeof(struct xxx) + alloc_elements_once: malloc elements once, 0 for malloc 1MB once +return error no, 0 for success, != 0 fail +*/ +int fast_mblock_init(struct fast_mblock_man *mblock, const int element_size, \ + const int alloc_elements_once); + +/** +mblock destroy +parameters: + mblock: the mblock pointer +*/ +void fast_mblock_destroy(struct fast_mblock_man *mblock); + +/** +alloc a node from the mblock +parameters: + mblock: the mblock pointer +return the alloced node, return NULL if fail +*/ +struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock); + +/** +free a node (put a node to the mblock) +parameters: + mblock: the mblock pointer + pNode: the node to free +return the alloced node, return NULL if fail +*/ +int fast_mblock_free(struct fast_mblock_man *mblock, \ + struct fast_mblock_node *pNode); + +/** +get node count of the mblock +parameters: + mblock: the mblock pointer +return the free node count of the mblock, return -1 if fail +*/ +int fast_mblock_count(struct fast_mblock_man *mblock); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/fast_task_queue.c b/common/fast_task_queue.c new file mode 100644 index 0000000..b55bc46 --- /dev/null +++ b/common/fast_task_queue.c @@ -0,0 +1,503 @@ +//fast_task_queue.c + +#include +#include +#include +#include "fast_task_queue.h" +#include "logger.h" +#include "shared_func.h" +#include "pthread_func.h" + +static struct fast_task_queue g_free_queue; + +struct mpool_chain { + struct fast_task_info *blocks; + struct fast_task_info *last_block; //last block + struct mpool_chain *next; +} *g_mpool = NULL; + +#define ALIGNED_TASK_INFO_SIZE MEM_ALIGN(sizeof(struct fast_task_info)) + +int task_queue_init(struct fast_task_queue *pQueue) +{ + int result; + + if ((result=init_pthread_lock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_lock fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + pQueue->head = NULL; + pQueue->tail = NULL; + + return 0; +} + +static struct mpool_chain *malloc_mpool(const int block_size, \ + const int total_alloc_size) +{ + struct fast_task_info *pTask; + char *p; + char *pCharEnd; + struct mpool_chain *mpool; + + mpool = (struct mpool_chain *)malloc(sizeof(struct mpool_chain)); + if (mpool == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(struct mpool_chain), \ + errno, STRERROR(errno)); + return NULL; + } + + mpool->next = NULL; + mpool->blocks = (struct fast_task_info *)malloc(total_alloc_size); + if (mpool->blocks == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, total_alloc_size, \ + errno, STRERROR(errno)); + free(mpool); + return NULL; + } + memset(mpool->blocks, 0, total_alloc_size); + + pCharEnd = ((char *)mpool->blocks) + total_alloc_size; + for (p=(char *)mpool->blocks; psize = g_free_queue.min_buff_size; + + pTask->arg = p + ALIGNED_TASK_INFO_SIZE; + if (g_free_queue.malloc_whole_block) + { + pTask->data = (char *)pTask->arg + \ + g_free_queue.arg_size; + } + else + { + pTask->data = (char *)malloc(pTask->size); + if (pTask->data == NULL) + { + char *pt; + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->size, \ + errno, STRERROR(errno)); + + for (pt=(char *)mpool->blocks; pt < p; \ + pt += block_size) + { + free(((struct fast_task_info *)pt)->data); + } + + free(mpool->blocks); + free(mpool); + return NULL; + } + } + } + + mpool->last_block = (struct fast_task_info *)(pCharEnd - block_size); + for (p=(char *)mpool->blocks; p<(char *)mpool->last_block; p += block_size) + { + pTask = (struct fast_task_info *)p; + pTask->next = (struct fast_task_info *)(p + block_size); + } + mpool->last_block->next = NULL; + + return mpool; +} + +int free_queue_init(const int max_connections, const int min_buff_size, \ + const int max_buff_size, const int arg_size) +{ + int64_t total_size; + struct mpool_chain *mpool; + int block_size; + int alloc_size; + int result; + int loop_count; + int aligned_min_size; + int aligned_max_size; + int aligned_arg_size; + rlim_t max_data_size; + + if ((result=init_pthread_lock(&(g_free_queue.lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_lock fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + aligned_min_size = MEM_ALIGN(min_buff_size); + aligned_max_size = MEM_ALIGN(max_buff_size); + aligned_arg_size = MEM_ALIGN(arg_size); + block_size = ALIGNED_TASK_INFO_SIZE + aligned_arg_size; + alloc_size = block_size * max_connections; + if (aligned_max_size > aligned_min_size) + { + total_size = alloc_size; + g_free_queue.malloc_whole_block = false; + max_data_size = 0; + } + else + { + struct rlimit rlimit_data; + + if (getrlimit(RLIMIT_DATA, &rlimit_data) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "call getrlimit fail, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + if (rlimit_data.rlim_cur == RLIM_INFINITY) + { + max_data_size = 256 * 1024 * 1024; + } + else + { + max_data_size = rlimit_data.rlim_cur; + if (max_data_size > 256 * 1024 * 1024) + { + max_data_size = 256 * 1024 * 1024; + } + } + + if (max_data_size >= (int64_t)(block_size + aligned_min_size) * + (int64_t)max_connections) + { + total_size = alloc_size + (int64_t)aligned_min_size * + max_connections; + g_free_queue.malloc_whole_block = true; + block_size += aligned_min_size; + } + else + { + total_size = alloc_size; + g_free_queue.malloc_whole_block = false; + max_data_size = 0; + } + } + + logDebug("file: "__FILE__", line: %d, " \ + "max_connections: %d, min_buff_size: %d, max_buff_size: %d, " \ + "block_size: %d, arg_size: %d, max_data_size: %d, " \ + "total_size: "INT64_PRINTF_FORMAT, __LINE__, \ + max_connections, aligned_min_size, aligned_max_size, \ + block_size, aligned_arg_size, (int)max_data_size, total_size); + + g_free_queue.max_connections = max_connections; + g_free_queue.min_buff_size = aligned_min_size; + g_free_queue.max_buff_size = aligned_max_size; + g_free_queue.arg_size = aligned_arg_size; + + if ((!g_free_queue.malloc_whole_block) || \ + (total_size <= max_data_size)) + { + loop_count = 1; + mpool = malloc_mpool(block_size, total_size); + if (mpool == NULL) + { + return errno != 0 ? errno : ENOMEM; + } + g_mpool = mpool; + } + else + { + struct mpool_chain *previous_mpool; + int remain_count; + int alloc_once; + int current_count; + int current_alloc_size; + + mpool = NULL; + previous_mpool = NULL; + loop_count = 0; + remain_count = max_connections; + alloc_once = max_data_size / block_size; + while (remain_count > 0) + { + current_count = (remain_count > alloc_once) ? \ + alloc_once : remain_count; + current_alloc_size = block_size * current_count; + mpool = malloc_mpool(block_size, current_alloc_size); + if (mpool == NULL) + { + free_queue_destroy(); + return errno != 0 ? errno : ENOMEM; + } + + if (previous_mpool == NULL) + { + g_mpool = mpool; + } + else + { + previous_mpool->next = mpool; + previous_mpool->last_block->next = mpool->blocks; + } + previous_mpool = mpool; + + remain_count -= current_count; + loop_count++; + } + + logDebug("file: "__FILE__", line: %d, " \ + "alloc_once: %d", __LINE__, alloc_once); + } + + logDebug("file: "__FILE__", line: %d, " \ + "malloc task info as whole: %d, malloc loop count: %d", \ + __LINE__, g_free_queue.malloc_whole_block, loop_count); + + if (g_mpool != NULL) + { + g_free_queue.head = g_mpool->blocks; + g_free_queue.tail = mpool->last_block; + + /* + struct fast_task_info *pTask; + int task_count = 0; + + pTask = g_free_queue.head; + while (pTask != NULL) + { + task_count++; + pTask = pTask->next; + } + logDebug("file: "__FILE__", line: %d, " \ + "task count: %d", __LINE__, task_count); + */ + } + + return 0; +} + +void free_queue_destroy() +{ + struct mpool_chain *mpool; + struct mpool_chain *mp; + + if (g_mpool == NULL) + { + return; + } + + if (!g_free_queue.malloc_whole_block) + { + char *p; + char *pCharEnd; + int block_size; + struct fast_task_info *pTask; + + block_size = ALIGNED_TASK_INFO_SIZE + g_free_queue.arg_size; + pCharEnd = ((char *)g_mpool->blocks) + block_size * \ + g_free_queue.max_connections; + for (p=(char *)g_mpool->blocks; pdata != NULL) + { + free(pTask->data); + pTask->data = NULL; + } + } + } + + mpool = g_mpool; + while (mpool != NULL) + { + mp = mpool; + mpool = mpool->next; + + free(mp->blocks); + free(mp); + } + g_mpool = NULL; + + pthread_mutex_destroy(&(g_free_queue.lock)); +} + +struct fast_task_info *free_queue_pop() +{ + return task_queue_pop(&g_free_queue);; +} + +int free_queue_push(struct fast_task_info *pTask) +{ + char *new_buff; + int result; + + *(pTask->client_ip) = '\0'; + pTask->length = 0; + pTask->offset = 0; + pTask->req_count = 0; + + if (pTask->size > g_free_queue.min_buff_size) //need thrink + { + new_buff = (char *)malloc(g_free_queue.min_buff_size); + if (new_buff == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, g_free_queue.min_buff_size, \ + errno, STRERROR(errno)); + } + else + { + free(pTask->data); + pTask->size = g_free_queue.min_buff_size; + pTask->data = new_buff; + } + } + + if ((result=pthread_mutex_lock(&g_free_queue.lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + pTask->next = g_free_queue.head; + g_free_queue.head = pTask; + if (g_free_queue.tail == NULL) + { + g_free_queue.tail = pTask; + } + + if ((result=pthread_mutex_unlock(&g_free_queue.lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return result; +} + +int free_queue_count() +{ + return task_queue_count(&g_free_queue); +} + +int task_queue_push(struct fast_task_queue *pQueue, \ + struct fast_task_info *pTask) +{ + int result; + + if ((result=pthread_mutex_lock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + pTask->next = NULL; + if (pQueue->tail == NULL) + { + pQueue->head = pTask; + } + else + { + pQueue->tail->next = pTask; + } + pQueue->tail = pTask; + + if ((result=pthread_mutex_unlock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return 0; +} + +struct fast_task_info *task_queue_pop(struct fast_task_queue *pQueue) +{ + struct fast_task_info *pTask; + int result; + + if ((result=pthread_mutex_lock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return NULL; + } + + pTask = pQueue->head; + if (pTask != NULL) + { + pQueue->head = pTask->next; + if (pQueue->head == NULL) + { + pQueue->tail = NULL; + } + } + + if ((result=pthread_mutex_unlock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return pTask; +} + +int task_queue_count(struct fast_task_queue *pQueue) +{ + struct fast_task_info *pTask; + int count; + int result; + + if ((result=pthread_mutex_lock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return 0; + } + + count = 0; + pTask = pQueue->head; + while (pTask != NULL) + { + pTask = pTask->next; + count++; + } + + if ((result=pthread_mutex_unlock(&(pQueue->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return count; +} + diff --git a/common/fast_task_queue.h b/common/fast_task_queue.h new file mode 100644 index 0000000..695ee25 --- /dev/null +++ b/common/fast_task_queue.h @@ -0,0 +1,96 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fast_task_queue.h + +#ifndef _FAST_TASK_QUEUE_H +#define _FAST_TASK_QUEUE_H + +#include +#include +#include +#include +#include "common_define.h" +#include "ioevent.h" +#include "fast_timer.h" + +struct fast_task_info; + +typedef int (*TaskFinishCallBack) (struct fast_task_info *pTask); +typedef void (*TaskCleanUpCallBack) (struct fast_task_info *pTask); + +typedef void (*IOEventCallback) (int sock, short event, void *arg); + +typedef struct ioevent_entry +{ + int fd; + FastTimerEntry timer; + IOEventCallback callback; +} IOEventEntry; + +struct nio_thread_data +{ + struct ioevent_puller ev_puller; + struct fast_timer timer; + int pipe_fds[2]; + struct fast_task_info *deleted_list; +}; + +struct fast_task_info +{ + IOEventEntry event; + char client_ip[IP_ADDRESS_SIZE]; + void *arg; //extra argument pointer + char *data; //buffer for write or recv + int size; //alloc size + int length; //data length + int offset; //current offset + int req_count; //request count + TaskFinishCallBack finish_callback; + struct nio_thread_data *thread_data; + struct fast_task_info *next; +}; + +struct fast_task_queue +{ + struct fast_task_info *head; + struct fast_task_info *tail; + pthread_mutex_t lock; + int max_connections; + int min_buff_size; + int max_buff_size; + int arg_size; + bool malloc_whole_block; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +int free_queue_init(const int max_connections, const int min_buff_size, \ + const int max_buff_size, const int arg_size); + +void free_queue_destroy(); + +int free_queue_push(struct fast_task_info *pTask); +struct fast_task_info *free_queue_pop(); +int free_queue_count(); + + +int task_queue_init(struct fast_task_queue *pQueue); +int task_queue_push(struct fast_task_queue *pQueue, \ + struct fast_task_info *pTask); +struct fast_task_info *task_queue_pop(struct fast_task_queue *pQueue); +int task_queue_count(struct fast_task_queue *pQueue); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/fast_timer.c b/common/fast_timer.c new file mode 100644 index 0000000..8462786 --- /dev/null +++ b/common/fast_timer.c @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include +#include "logger.h" +#include "fast_timer.h" + +int fast_timer_init(FastTimer *timer, const int slot_count, + const int64_t current_time) +{ + int bytes; + if (slot_count <= 0 || current_time <= 0) { + return EINVAL; + } + + timer->slot_count = slot_count; + timer->base_time = current_time; //base time for slot 0 + timer->current_time = current_time; + bytes = sizeof(FastTimerSlot) * slot_count; + timer->slots = (FastTimerSlot *)malloc(bytes); + if (timer->slots == NULL) { + return errno != 0 ? errno : ENOMEM; + } + memset(timer->slots, 0, bytes); + return 0; +} + +void fast_timer_destroy(FastTimer *timer) +{ + if (timer->slots != NULL) { + free(timer->slots); + timer->slots = NULL; + } +} + +#define TIMER_GET_SLOT_INDEX(timer, expires) \ + (((expires) - timer->base_time) % timer->slot_count) + +#define TIMER_GET_SLOT_POINTER(timer, expires) \ + (timer->slots + TIMER_GET_SLOT_INDEX(timer, expires)) + +int fast_timer_add(FastTimer *timer, FastTimerEntry *entry) +{ + FastTimerSlot *slot; + + slot = TIMER_GET_SLOT_POINTER(timer, entry->expires > + timer->current_time ? entry->expires : timer->current_time); + entry->next = slot->head.next; + if (slot->head.next != NULL) { + slot->head.next->prev = entry; + } + entry->prev = &slot->head; + slot->head.next = entry; + return 0; +} + +int fast_timer_modify(FastTimer *timer, FastTimerEntry *entry, + const int64_t new_expires) +{ + if (new_expires == entry->expires) { + return 0; + } + + if (new_expires < entry->expires) { + fast_timer_remove(timer, entry); + entry->expires = new_expires; + return fast_timer_add(timer, entry); + } + + entry->rehash = TIMER_GET_SLOT_INDEX(timer, new_expires) != + TIMER_GET_SLOT_INDEX(timer, entry->expires); + entry->expires = new_expires; //lazy move + return 0; +} + +int fast_timer_remove(FastTimer *timer, FastTimerEntry *entry) +{ + if (entry->prev == NULL) { + return ENOENT; //already removed + } + + if (entry->next != NULL) { + entry->next->prev = entry->prev; + entry->prev->next = entry->next; + entry->next = NULL; + } + else { + entry->prev->next = NULL; + } + + entry->prev = NULL; + return 0; +} + +FastTimerSlot *fast_timer_slot_get(FastTimer *timer, const int64_t current_time) +{ + if (timer->current_time >= current_time) { + return NULL; + } + + return TIMER_GET_SLOT_POINTER(timer, timer->current_time++); +} + +int fast_timer_timeouts_get(FastTimer *timer, const int64_t current_time, + FastTimerEntry *head) +{ + FastTimerSlot *slot; + FastTimerEntry *entry; + FastTimerEntry *first; + FastTimerEntry *last; + FastTimerEntry *tail; + int count; + + head->prev = NULL; + head->next = NULL; + if (timer->current_time >= current_time) { + return 0; + } + + first = NULL; + last = NULL; + tail = head; + count = 0; + while (timer->current_time < current_time) { + slot = TIMER_GET_SLOT_POINTER(timer, timer->current_time++); + entry = slot->head.next; + while (entry != NULL) { + if (entry->expires >= current_time) { //not expired + if (first != NULL) { + first->prev->next = entry; + entry->prev = first->prev; + + tail->next = first; + first->prev = tail; + tail = last; + first = NULL; + } + if (entry->rehash) { + last = entry; + entry = entry->next; + + last->rehash = false; + fast_timer_remove(timer, last); + fast_timer_add(timer, last); + continue; + } + } + else { + count++; + if (first == NULL) { + first = entry; + } + } + last = entry; + entry = entry->next; + } + + if (first != NULL) { + first->prev->next = NULL; + + tail->next = first; + first->prev = tail; + tail = last; + first = NULL; + } + } + + if (count > 0) { + tail->next = NULL; + } + + return count; +} + diff --git a/common/fast_timer.h b/common/fast_timer.h new file mode 100644 index 0000000..a3e3e39 --- /dev/null +++ b/common/fast_timer.h @@ -0,0 +1,48 @@ +#ifndef __FAST_TIMER_H__ +#define __FAST_TIMER_H__ + +#include +#include "common_define.h" + +typedef struct fast_timer_entry { + int64_t expires; + void *data; + struct fast_timer_entry *prev; + struct fast_timer_entry *next; + bool rehash; +} FastTimerEntry; + +typedef struct fast_timer_slot { + struct fast_timer_entry head; +} FastTimerSlot; + +typedef struct fast_timer { + int slot_count; //time wheel slot count + int64_t base_time; //base time for slot 0 + int64_t current_time; + FastTimerSlot *slots; +} FastTimer; + +#ifdef __cplusplus +extern "C" { +#endif + +int fast_timer_init(FastTimer *timer, const int slot_count, + const int64_t current_time); +void fast_timer_destroy(FastTimer *timer); + +int fast_timer_add(FastTimer *timer, FastTimerEntry *entry); +int fast_timer_remove(FastTimer *timer, FastTimerEntry *entry); +int fast_timer_modify(FastTimer *timer, FastTimerEntry *entry, + const int64_t new_expires); + +FastTimerSlot *fast_timer_slot_get(FastTimer *timer, const int64_t current_time); +int fast_timer_timeouts_get(FastTimer *timer, const int64_t current_time, + FastTimerEntry *head); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/fdfs_define.h b/common/fdfs_define.h new file mode 100644 index 0000000..2eaf394 --- /dev/null +++ b/common/fdfs_define.h @@ -0,0 +1,42 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdfs_define.h + +#ifndef _FDFS_DEFINE_H_ +#define _FDFS_DEFINE_H_ + +#include +#include "common_define.h" + +#define FDFS_TRACKER_SERVER_DEF_PORT 22000 +#define FDFS_STORAGE_SERVER_DEF_PORT 23000 +#define FDFS_DEF_STORAGE_RESERVED_MB 1024 +#define TRACKER_ERROR_LOG_FILENAME "trackerd" +#define STORAGE_ERROR_LOG_FILENAME "storaged" + +#define FDFS_RECORD_SEPERATOR '\x01' +#define FDFS_FIELD_SEPERATOR '\x02' + +#define SYNC_BINLOG_BUFF_DEF_INTERVAL 60 +#define CHECK_ACTIVE_DEF_INTERVAL 100 + +#define DEFAULT_STORAGE_SYNC_FILE_MAX_DELAY 86400 +#define DEFAULT_STORAGE_SYNC_FILE_MAX_TIME 300 + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/fdfs_global.c b/common/fdfs_global.c new file mode 100644 index 0000000..c4d46d4 --- /dev/null +++ b/common/fdfs_global.c @@ -0,0 +1,149 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "fdfs_global.h" + +int g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT; +int g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT; +char g_fdfs_base_path[MAX_PATH_SIZE] = {'/', 't', 'm', 'p', '\0'}; +Version g_fdfs_version = {5, 2}; +bool g_use_connection_pool = false; +ConnectionPool g_connection_pool; +int g_connection_pool_max_idle_time = 3600; + +/* +data filename format: +HH/HH/filename: HH for 2 uppercase hex chars +*/ +int fdfs_check_data_filename(const char *filename, const int len) +{ + if (len < 6) + { + logError("file: "__FILE__", line: %d, " \ + "the length=%d of filename \"%s\" is too short", \ + __LINE__, len, filename); + return EINVAL; + } + + if (!IS_UPPER_HEX(*filename) || !IS_UPPER_HEX(*(filename+1)) || \ + *(filename+2) != '/' || \ + !IS_UPPER_HEX(*(filename+3)) || !IS_UPPER_HEX(*(filename+4)) || \ + *(filename+5) != '/') + { + logError("file: "__FILE__", line: %d, " \ + "the format of filename \"%s\" is invalid", \ + __LINE__, filename); + return EINVAL; + } + + if (strchr(filename + 6, '/') != NULL) + { + logError("file: "__FILE__", line: %d, " \ + "the format of filename \"%s\" is invalid", \ + __LINE__, filename); + return EINVAL; + } + + return 0; +} + +int fdfs_gen_slave_filename(const char *master_filename, \ + const char *prefix_name, const char *ext_name, \ + char *filename, int *filename_len) +{ + char true_ext_name[FDFS_FILE_EXT_NAME_MAX_LEN + 2]; + char *pDot; + int master_file_len; + + master_file_len = strlen(master_filename); + if (master_file_len < 28 + FDFS_FILE_EXT_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "master filename \"%s\" is invalid", \ + __LINE__, master_filename); + return EINVAL; + } + + pDot = strchr(master_filename + (master_file_len - \ + (FDFS_FILE_EXT_NAME_MAX_LEN + 1)), '.'); + if (ext_name != NULL) + { + if (*ext_name == '\0') + { + *true_ext_name = '\0'; + } + else if (*ext_name == '.') + { + snprintf(true_ext_name, sizeof(true_ext_name), \ + "%s", ext_name); + } + else + { + snprintf(true_ext_name, sizeof(true_ext_name), \ + ".%s", ext_name); + } + } + else + { + if (pDot == NULL) + { + *true_ext_name = '\0'; + } + else + { + strcpy(true_ext_name, pDot); + } + } + + if (*true_ext_name == '\0' && strcmp(prefix_name, "-m") == 0) + { + logError("file: "__FILE__", line: %d, " \ + "prefix_name \"%s\" is invalid", \ + __LINE__, prefix_name); + return EINVAL; + } + + /* when prefix_name is empty, the extension name of master file and + slave file can not be same + */ + if ((*prefix_name == '\0') && ((pDot == NULL && *true_ext_name == '\0') + || (pDot != NULL && strcmp(pDot, true_ext_name) == 0))) + { + logError("file: "__FILE__", line: %d, " \ + "empty prefix_name is not allowed", __LINE__); + return EINVAL; + } + + if (pDot == NULL) + { + *filename_len = sprintf(filename, "%s%s%s", master_filename, \ + prefix_name, true_ext_name); + } + else + { + *filename_len = pDot - master_filename; + memcpy(filename, master_filename, *filename_len); + *filename_len += sprintf(filename + *filename_len, "%s%s", \ + prefix_name, true_ext_name); + } + + return 0; +} + diff --git a/common/fdfs_global.h b/common/fdfs_global.h new file mode 100644 index 0000000..8aae997 --- /dev/null +++ b/common/fdfs_global.h @@ -0,0 +1,42 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdfs_global.h + +#ifndef _FDFS_GLOBAL_H +#define _FDFS_GLOBAL_H + +#include "common_define.h" +#include "fdfs_define.h" +#include "connection_pool.h" + +#define FDFS_FILE_EXT_NAME_MAX_LEN 6 + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_fdfs_connect_timeout; +extern int g_fdfs_network_timeout; +extern char g_fdfs_base_path[MAX_PATH_SIZE]; +extern Version g_fdfs_version; +extern bool g_use_connection_pool; +extern ConnectionPool g_connection_pool; +extern int g_connection_pool_max_idle_time; + +int fdfs_check_data_filename(const char *filename, const int len); +int fdfs_gen_slave_filename(const char *master_filename, \ + const char *prefix_name, const char *ext_name, \ + char *filename, int *filename_len); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/fdfs_http_shared.c b/common/fdfs_http_shared.c new file mode 100644 index 0000000..66a754b --- /dev/null +++ b/common/fdfs_http_shared.c @@ -0,0 +1,372 @@ + +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "md5.h" +#include "shared_func.h" +#include "mime_file_parser.h" +#include "fdfs_global.h" +#include "fdfs_http_shared.h" + +const char *fdfs_http_get_file_extension(const char *filename, \ + const int filename_len, int *ext_len) +{ + const char *pEnd; + const char *pExtName; + int i; + + pEnd = filename + filename_len; + pExtName = pEnd - 1; + for (i=0; i= filename; \ + i++, pExtName--) + { + if (*pExtName == '.') + { + break; + } + } + + if (i < FDFS_FILE_EXT_NAME_MAX_LEN) //found + { + pExtName++; //skip . + *ext_len = pEnd - pExtName; + return pExtName; + } + else + { + *ext_len = 0; + return NULL; + } +} + +int fdfs_http_get_content_type_by_extname(FDFSHTTPParams *pParams, \ + const char *ext_name, const int ext_len, \ + char *content_type, const int content_type_size) +{ + HashData *pHashData; + + if (ext_len == 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "extension name is empty, " \ + "set to default content type: %s", \ + __LINE__, pParams->default_content_type); + strcpy(content_type, pParams->default_content_type); + return 0; + } + + pHashData = hash_find_ex(&pParams->content_type_hash, \ + ext_name, ext_len + 1); + if (pHashData == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "extension name: %s is not supported, " \ + "set to default content type: %s", \ + __LINE__, ext_name, pParams->default_content_type); + strcpy(content_type, pParams->default_content_type); + return 0; + } + + if (pHashData->value_len >= content_type_size) + { + *content_type = '\0'; + logError("file: "__FILE__", line: %d, " \ + "extension name: %s 's content type " \ + "is too long", __LINE__, ext_name); + return EINVAL; + } + + memcpy(content_type, pHashData->value, pHashData->value_len); + return 0; +} + +int fdfs_http_params_load(IniContext *pIniContext, \ + const char *conf_filename, FDFSHTTPParams *pParams) +{ + int result; + int ext_len; + const char *ext_name; + char *mime_types_filename; + char szMimeFilename[256]; + char *anti_steal_secret_key; + char *token_check_fail_filename; + char *default_content_type; + int def_content_type_len; + int64_t file_size; + + memset(pParams, 0, sizeof(FDFSHTTPParams)); + + pParams->disabled = iniGetBoolValue(NULL, "http.disabled", \ + pIniContext, false); + if (pParams->disabled) + { + return 0; + } + + pParams->need_find_content_type = iniGetBoolValue(NULL, \ + "http.need_find_content_type", \ + pIniContext, true); + + pParams->server_port = iniGetIntValue(NULL, "http.server_port", \ + pIniContext, 80); + if (pParams->server_port <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid param \"http.server_port\": %d", \ + __LINE__, pParams->server_port); + return EINVAL; + } + + pParams->anti_steal_token = iniGetBoolValue(NULL, \ + "http.anti_steal.check_token", \ + pIniContext, false); + if (pParams->need_find_content_type || pParams->anti_steal_token) + { + mime_types_filename = iniGetStrValue(NULL, "http.mime_types_filename", \ + pIniContext); + if (mime_types_filename == NULL || *mime_types_filename == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "param \"http.mime_types_filename\" not exist " \ + "or is empty", __LINE__); + return EINVAL; + } + + if (strncasecmp(mime_types_filename, "http://", 7) != 0 && \ + *mime_types_filename != '/' && \ + strncasecmp(conf_filename, "http://", 7) != 0) + { + char *pPathEnd; + + pPathEnd = strrchr(conf_filename, '/'); + if (pPathEnd == NULL) + { + snprintf(szMimeFilename, sizeof(szMimeFilename), \ + "%s", mime_types_filename); + } + else + { + int nPathLen; + int nFilenameLen; + + nPathLen = (pPathEnd - conf_filename) + 1; + nFilenameLen = strlen(mime_types_filename); + if (nPathLen + nFilenameLen >= sizeof(szMimeFilename)) + { + logError("file: "__FILE__", line: %d, " \ + "filename is too long, length %d >= %d", + __LINE__, nPathLen + nFilenameLen, \ + (int)sizeof(szMimeFilename)); + return ENOSPC; + } + + memcpy(szMimeFilename, conf_filename, nPathLen); + memcpy(szMimeFilename + nPathLen, mime_types_filename, \ + nFilenameLen); + *(szMimeFilename + nPathLen + nFilenameLen) = '\0'; + } + } + else + { + snprintf(szMimeFilename, sizeof(szMimeFilename), \ + "%s", mime_types_filename); + } + + result = load_mime_types_from_file(&pParams->content_type_hash, \ + szMimeFilename); + if (result != 0) + { + return result; + } + + default_content_type = iniGetStrValue(NULL, \ + "http.default_content_type", \ + pIniContext); + if (default_content_type == NULL || *default_content_type == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "param \"http.default_content_type\" not exist " \ + "or is empty", __LINE__); + return EINVAL; + } + + def_content_type_len = strlen(default_content_type); + if (def_content_type_len >= sizeof(pParams->default_content_type)) + { + logError("file: "__FILE__", line: %d, " \ + "default content type: %s is too long", \ + __LINE__, default_content_type); + return EINVAL; + } + memcpy(pParams->default_content_type, default_content_type, \ + def_content_type_len); + } + + if (!pParams->anti_steal_token) + { + return 0; + } + + pParams->token_ttl = iniGetIntValue(NULL, \ + "http.anti_steal.token_ttl", \ + pIniContext, 600); + if (pParams->token_ttl <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "param \"http.anti_steal.token_ttl\" is invalid", \ + __LINE__); + return EINVAL; + } + + anti_steal_secret_key = iniGetStrValue(NULL, \ + "http.anti_steal.secret_key", \ + pIniContext); + if (anti_steal_secret_key == NULL || *anti_steal_secret_key == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "param \"http.anti_steal.secret_key\" not exist " \ + "or is empty", __LINE__); + return EINVAL; + } + + buffer_strcpy(&pParams->anti_steal_secret_key, anti_steal_secret_key); + + token_check_fail_filename = iniGetStrValue(NULL, \ + "http.anti_steal.token_check_fail", \ + pIniContext); + if (token_check_fail_filename == NULL || \ + *token_check_fail_filename == '\0') + { + return 0; + } + + if (!fileExists(token_check_fail_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "token_check_fail file: %s not exists", __LINE__, \ + token_check_fail_filename); + return ENOENT; + } + + ext_name = fdfs_http_get_file_extension(token_check_fail_filename, \ + strlen(token_check_fail_filename), &ext_len); + if ((result=fdfs_http_get_content_type_by_extname(pParams, \ + ext_name, ext_len, \ + pParams->token_check_fail_content_type, \ + sizeof(pParams->token_check_fail_content_type))) != 0) + { + return result; + } + + if (!pParams->need_find_content_type) + { + hash_destroy(&pParams->content_type_hash); + } + + if ((result=getFileContent(token_check_fail_filename, \ + &pParams->token_check_fail_buff.buff, &file_size)) != 0) + { + return result; + } + + pParams->token_check_fail_buff.alloc_size = file_size; + pParams->token_check_fail_buff.length = file_size; + + return 0; +} + +void fdfs_http_params_destroy(FDFSHTTPParams *pParams) +{ + if (pParams->need_find_content_type) + { + hash_destroy(&pParams->content_type_hash); + } +} + +int fdfs_http_gen_token(const BufferInfo *secret_key, const char *file_id, \ + const int timestamp, char *token) +{ + char buff[256 + 64]; + unsigned char digit[16]; + int id_len; + int total_len; + + id_len = strlen(file_id); + if (id_len + secret_key->length + 12 > sizeof(buff)) + { + return ENOSPC; + } + + memcpy(buff, file_id, id_len); + total_len = id_len; + memcpy(buff + total_len, secret_key->buff, secret_key->length); + total_len += secret_key->length; + total_len += sprintf(buff + total_len, "%d", timestamp); + + my_md5_buffer(buff, total_len, digit); + bin2hex((char *)digit, 16, token); + return 0; +} + +int fdfs_http_check_token(const BufferInfo *secret_key, const char *file_id, \ + const int timestamp, const char *token, const int ttl) +{ + char true_token[33]; + int result; + int token_len; + + token_len = strlen(token); + if (token_len != 32) + { + return EINVAL; + } + + if ((timestamp != 0) && (time(NULL) - timestamp > ttl)) + { + return ETIMEDOUT; + } + + if ((result=fdfs_http_gen_token(secret_key, file_id, \ + timestamp, true_token)) != 0) + { + return result; + } + + return (memcmp(token, true_token, 32) == 0) ? 0 : EPERM; +} + +char *fdfs_http_get_parameter(const char *param_name, KeyValuePair *params, \ + const int param_count) +{ + KeyValuePair *pCurrent; + KeyValuePair *pEnd; + + pEnd = params + param_count; + for (pCurrent=params; pCurrentkey, param_name) == 0) + { + return pCurrent->value; + } + } + + return NULL; +} + diff --git a/common/fdfs_http_shared.h b/common/fdfs_http_shared.h new file mode 100644 index 0000000..14e111f --- /dev/null +++ b/common/fdfs_http_shared.h @@ -0,0 +1,125 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef _FDFS_HTTP_SHARED_H +#define _FDFS_HTTP_SHARED_H + +#include +#include +#include +#include +#include +#include "ini_file_reader.h" +#include "hash.h" + +typedef struct +{ + bool disabled; + bool anti_steal_token; + + /* if need find content type by file extension name */ + bool need_find_content_type; + + /* the web server port */ + int server_port; + + /* key is file ext name, value is content type */ + HashArray content_type_hash; + + BufferInfo anti_steal_secret_key; + BufferInfo token_check_fail_buff; + char default_content_type[64]; + char token_check_fail_content_type[64]; + int token_ttl; +} FDFSHTTPParams; + +#ifdef __cplusplus +extern "C" { +#endif + +/** +load HTTP params from conf file +params: + pIniContext: the ini file items, return by iniLoadItems + conf_filename: config filename + pHTTPParams: the HTTP params +return: 0 for success, != 0 fail +**/ +int fdfs_http_params_load(IniContext *pIniContext, \ + const char *conf_filename, FDFSHTTPParams *pHTTPParams); + +void fdfs_http_params_destroy(FDFSHTTPParams *pParams); + +/** +generate anti-steal token +params: + secret_key: secret key buffer + file_id: FastDFS file id + timestamp: current timestamp, unix timestamp (seconds), 0 for never timeout + token: return token buffer +return: 0 for success, != 0 fail +**/ +int fdfs_http_gen_token(const BufferInfo *secret_key, const char *file_id, \ + const int timestamp, char *token); + +/** +check anti-steal token +params: + secret_key: secret key buffer + file_id: FastDFS file id + timestamp: the timestamp to generate the token, unix timestamp (seconds) + token: token buffer + ttl: token ttl, delta seconds +return: 0 for passed, != 0 fail +**/ +int fdfs_http_check_token(const BufferInfo *secret_key, const char *file_id, \ + const int timestamp, const char *token, const int ttl); + +/** +get parameter value +params: + param_name: the parameter name to get + params: parameter array + param_count: param count +return: param value pointer, return NULL if not exist +**/ +char *fdfs_http_get_parameter(const char *param_name, KeyValuePair *params, \ + const int param_count); + + +/** +get file extension name +params: + filename: the filename + filename_len: the length of filename + ext_len: return the length of extension name +return: extension name, NULL for none +**/ +const char *fdfs_http_get_file_extension(const char *filename, \ + const int filename_len, int *ext_len); + +/** +get content type by file extension name +params: + pHTTPParams: the HTTP params + ext_name: the extension name + ext_len: the length of extension name + content_type: return content type + content_type_size: content type buffer size +return: 0 for success, != 0 fail +**/ +int fdfs_http_get_content_type_by_extname(FDFSHTTPParams *pParams, \ + const char *ext_name, const int ext_len, \ + char *content_type, const int content_type_size); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/hash.c b/common/hash.c new file mode 100644 index 0000000..b8a49d5 --- /dev/null +++ b/common/hash.c @@ -0,0 +1,1402 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include "pthread_func.h" +#include "hash.h" + +static unsigned int prime_array[] = { + 1, /* 0 */ + 3, /* 1 */ + 17, /* 2 */ + 37, /* 3 */ + 79, /* 4 */ + 163, /* 5 */ + 331, /* 6 */ + 673, /* 7 */ + 1361, /* 8 */ + 2729, /* 9 */ + 5471, /* 10 */ + 10949, /* 11 */ + 21911, /* 12 */ + 43853, /* 13 */ + 87719, /* 14 */ + 175447, /* 15 */ + 350899, /* 16 */ + 701819, /* 17 */ + 1403641, /* 18 */ + 2807303, /* 19 */ + 5614657, /* 20 */ + 11229331, /* 21 */ + 22458671, /* 22 */ + 44917381, /* 23 */ + 89834777, /* 24 */ + 179669557, /* 25 */ + 359339171, /* 26 */ + 718678369, /* 27 */ + 1437356741, /* 28 */ + 2147483647 /* 29 (largest signed int prime) */ +}; + +#define PRIME_ARRAY_SIZE 30 + +static int _hash_alloc_buckets(HashArray *pHash, const unsigned int old_capacity) +{ + size_t bytes; + + bytes = sizeof(HashData *) * (*pHash->capacity); + if (pHash->max_bytes > 0 && pHash->bytes_used+bytes > pHash->max_bytes) + { + return ENOSPC; + } + + pHash->buckets = (HashData **)malloc(bytes); + if (pHash->buckets == NULL) + { + return ENOMEM; + } + + memset(pHash->buckets, 0, bytes); + pHash->bytes_used += bytes - sizeof(HashData *) * old_capacity; + + return 0; +} + +int hash_init_ex(HashArray *pHash, HashFunc hash_func, \ + const unsigned int capacity, const double load_factor, \ + const int64_t max_bytes, const bool bMallocValue) +{ + unsigned int *pprime; + unsigned int *prime_end; + int result; + + memset(pHash, 0, sizeof(HashArray)); + prime_end = prime_array + PRIME_ARRAY_SIZE; + for (pprime = prime_array; pprime!=prime_end; pprime++) + { + if (*pprime > capacity) + { + pHash->capacity = pprime; + break; + } + } + + if (pHash->capacity == NULL) + { + return EINVAL; + } + + if ((result=_hash_alloc_buckets(pHash, 0)) != 0) + { + return result; + } + + pHash->hash_func = hash_func; + pHash->max_bytes = max_bytes; + pHash->is_malloc_value = bMallocValue; + + if (load_factor >= 0.00 && load_factor <= 1.00) + { + pHash->load_factor = load_factor; + } + else + { + pHash->load_factor = 0.50; + } + + return 0; +} + +int hash_set_locks(HashArray *pHash, const int lock_count) +{ + size_t bytes; + pthread_mutex_t *lock; + pthread_mutex_t *lock_end; + + if (pHash->locks != NULL) + { + return EEXIST; + } + + if (lock_count <= 0) + { + return EINVAL; + } + + if (pHash->load_factor >= 0.10) + { + return EINVAL; + } + + bytes = sizeof(pthread_mutex_t) * lock_count; + pHash->locks = (pthread_mutex_t *)malloc(bytes); + if (pHash->locks == NULL) + { + return ENOMEM; + } + + pHash->lock_count = lock_count; + lock_end = pHash->locks + lock_count; + for (lock=pHash->locks; lockbuckets == NULL) + { + return; + } + + bucket_end = pHash->buckets + (*pHash->capacity); + for (ppBucket=pHash->buckets; ppBucketnext; + free(pDelete); + } + } + + free(pHash->buckets); + pHash->buckets = NULL; + if (pHash->is_malloc_capacity) + { + free(pHash->capacity); + pHash->capacity = NULL; + pHash->is_malloc_capacity = false; + } + + pHash->item_count = 0; + pHash->bytes_used = 0; +} + +#define ADD_TO_BUCKET(pHash, ppBucket, hash_data) \ + hash_data->next = *ppBucket; \ + *ppBucket = hash_data; \ + pHash->item_count++; + + +#define DELETE_FROM_BUCKET(pHash, ppBucket, previous, hash_data) \ + if (previous == NULL) \ + { \ + *ppBucket = hash_data->next; \ + } \ + else \ + { \ + previous->next = hash_data->next; \ + } \ + pHash->item_count--; \ + pHash->bytes_used -= CALC_NODE_MALLOC_BYTES(hash_data->key_len, \ + hash_data->malloc_value_size); \ + free(hash_data); + +#define HASH_LOCK(pHash, index) \ + if (pHash->lock_count > 0) \ + { \ + pthread_mutex_lock(pHash->locks + (index) % pHash->lock_count); \ + } + +#define HASH_UNLOCK(pHash, index) \ + if (pHash->lock_count > 0) \ + { \ + pthread_mutex_unlock(pHash->locks + (index) % pHash->lock_count); \ + } + + +int hash_stat(HashArray *pHash, HashStat *pStat, \ + int *stat_by_lens, const int stat_size) +{ + HashData **ppBucket; + HashData **bucket_end; + HashData *hash_data; + int totalLength; + int last; + int count; + int i; + + memset(stat_by_lens, 0, sizeof(int) * stat_size); + pStat->bucket_max_length = 0; + pStat->bucket_used = 0; + last = stat_size - 1; + bucket_end = pHash->buckets + (*pHash->capacity); + for (ppBucket=pHash->buckets; ppBucketnext; + } + + pStat->bucket_used++; + if (count > last) + { + return ENOSPC; + } + stat_by_lens[count]++; + + if (count > pStat->bucket_max_length) + { + pStat->bucket_max_length = count; + } + } + + totalLength = 0; + for (i=0; i<=pStat->bucket_max_length; i++) + { + if (stat_by_lens[i] > 0) + { + totalLength += i * stat_by_lens[i]; + } + } + + pStat->capacity = *(pHash->capacity); + pStat->item_count = pHash->item_count; + pStat->bucket_avg_length = pStat->bucket_used > 0 ? \ + (double)totalLength / (double)pStat->bucket_used : 0.00; + + return 0; +} + +void hash_stat_print(HashArray *pHash) +{ +#define STAT_MAX_NUM 64 + HashStat hs; + int stats[STAT_MAX_NUM]; + + if (hash_stat(pHash, &hs, stats, STAT_MAX_NUM) != 0) + { + printf("hash max length exceeds %d!\n", STAT_MAX_NUM); + return; + } + + /* + printf("collision stat:\n"); + for (i=0; i 0) printf("%d: %d\n", i+1, stats[i]); + } + if (stats[i] > 0) printf(">=%d: %d\n", i+1, stats[i]); + */ + + printf("capacity: %d, item_count=%d, bucket_used: %d, " \ + "avg length: %.4f, max length: %d, bucket / item = %.2f%%\n", + hs.capacity, hs.item_count, hs.bucket_used, + hs.bucket_avg_length, hs.bucket_max_length, + (double)hs.bucket_used*100.00/(double)hs.capacity); +} + +static int _rehash1(HashArray *pHash, const int old_capacity, \ + unsigned int *new_capacity) +{ + HashData **old_buckets; + HashData **ppBucket; + HashData **bucket_end; + HashData *hash_data; + HashData *pNext; + int result; + + old_buckets = pHash->buckets; + pHash->capacity = new_capacity; + if ((result=_hash_alloc_buckets(pHash, old_capacity)) != 0) + { + pHash->buckets = old_buckets; + return result; + } + + //printf("old: %d, new: %d\n", old_capacity, *pHash->capacity); + + pHash->item_count = 0; + bucket_end = old_buckets + old_capacity; + for (ppBucket=old_buckets; ppBucketnext; + + ADD_TO_BUCKET(pHash, (pHash->buckets + \ + (HASH_CODE(pHash, hash_data) % \ + (*pHash->capacity))), hash_data) + + hash_data = pNext; + } + } + + free(old_buckets); + return 0; +} + +static int _rehash(HashArray *pHash) +{ + int result; + unsigned int *pOldCapacity; + + pOldCapacity = pHash->capacity; + if (pHash->is_malloc_capacity) + { + unsigned int *pprime; + unsigned int *prime_end; + + pHash->capacity = NULL; + + prime_end = prime_array + PRIME_ARRAY_SIZE; + for (pprime = prime_array; pprime!=prime_end; pprime++) + { + if (*pprime > *pOldCapacity) + { + pHash->capacity = pprime; + break; + } + } + } + else + { + pHash->capacity++; + } + + if ((result=_rehash1(pHash, *pOldCapacity, pHash->capacity)) != 0) + { + pHash->capacity = pOldCapacity; //rollback + } + else + { + if (pHash->is_malloc_capacity) + { + free(pOldCapacity); + pHash->is_malloc_capacity = false; + } + } + + /*printf("rehash, old_capacity=%d, new_capacity=%d\n", \ + old_capacity, *pHash->capacity); + */ + return result; +} + +static int _hash_conflict_count(HashArray *pHash) +{ + HashData **ppBucket; + HashData **bucket_end; + HashData *hash_data; + HashData *pNext; + int conflicted; + int conflict_count; + + bucket_end = pHash->buckets + (*pHash->capacity); + conflict_count = 0; + for (ppBucket=pHash->buckets; ppBucketnext == NULL) + { + continue; + } + + conflicted = 0; + hash_data = *ppBucket; + while (hash_data != NULL) + { + pNext = hash_data->next; + while (pNext != NULL) + { + if (HASH_CODE(pHash, hash_data) != \ + HASH_CODE(pHash, pNext)) + { + conflicted = 1; + break; + } + + pNext = pNext->next; + } + + if (conflicted) + { + break; + } + + hash_data = hash_data->next; + } + + conflict_count += conflicted; + } + + return conflict_count; +} + +int hash_best_op(HashArray *pHash, const int suggest_capacity) +{ + int old_capacity; + int conflict_count; + unsigned int *new_capacity; + int result; + + if ((conflict_count=_hash_conflict_count(pHash)) == 0) + { + return 0; + } + + old_capacity = *pHash->capacity; + new_capacity = (unsigned int *)malloc(sizeof(unsigned int)); + if (new_capacity == NULL) + { + return -ENOMEM; + } + + if ((suggest_capacity > 2) && (suggest_capacity >= pHash->item_count)) + { + *new_capacity = suggest_capacity - 2; + if (*new_capacity % 2 == 0) + { + ++(*new_capacity); + } + } + else + { + *new_capacity = 2 * (pHash->item_count - 1) + 1; + } + + do + { + do + { + *new_capacity += 2; + } while ((*new_capacity % 3 == 0) || (*new_capacity % 5 == 0) \ + || (*new_capacity % 7 == 0)); + + if ((result=_rehash1(pHash, old_capacity, new_capacity)) != 0) + { + pHash->is_malloc_capacity = \ + (pHash->capacity == new_capacity); + *pHash->capacity = old_capacity; + return -1 * result; + } + + old_capacity = *new_capacity; + /*printf("rehash, conflict_count=%d, old_capacity=%d, " \ + "new_capacity=%d\n", conflict_count, \ + old_capacity, *new_capacity); + */ + } while ((conflict_count=_hash_conflict_count(pHash)) > 0); + + pHash->is_malloc_capacity = true; + + //hash_stat_print(pHash); + return 1; +} + +static HashData *_chain_find_entry(HashData **ppBucket, const void *key, \ + const int key_len, const unsigned int hash_code) +{ + HashData *hash_data; + + hash_data = *ppBucket; + while (hash_data != NULL) + { + if (key_len == hash_data->key_len && \ + memcmp(key, hash_data->key, key_len) == 0) + { + return hash_data; + } + + hash_data = hash_data->next; + } + + return NULL; +} + +HashData *hash_find_ex(HashArray *pHash, const void *key, const int key_len) +{ + unsigned int hash_code; + HashData **ppBucket; + HashData *hash_data; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + HASH_LOCK(pHash, ppBucket - pHash->buckets) + hash_data = _chain_find_entry(ppBucket, key, key_len, hash_code); + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + + return hash_data; +} + +void *hash_find(HashArray *pHash, const void *key, const int key_len) +{ + unsigned int hash_code; + HashData **ppBucket; + HashData *hash_data; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + HASH_LOCK(pHash, ppBucket - pHash->buckets) + hash_data = _chain_find_entry(ppBucket, key, key_len, hash_code); + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + + if (hash_data != NULL) + { + return hash_data->value; + } + else + { + return NULL; + } +} + +int hash_get(HashArray *pHash, const void *key, const int key_len, + void *value, int *value_len) +{ + unsigned int hash_code; + int result; + HashData **ppBucket; + HashData *hash_data; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + HASH_LOCK(pHash, ppBucket - pHash->buckets) + hash_data = _chain_find_entry(ppBucket, key, key_len, hash_code); + if (hash_data != NULL) + { + if (hash_data->value_len <= *value_len) + { + *value_len = hash_data->value_len; + memcpy(value, hash_data->value, hash_data->value_len); + result = 0; + } + else + { + result = ENOSPC; + } + } + else + { + result = ENOENT; + } + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + return result; +} + +int hash_insert_ex(HashArray *pHash, const void *key, const int key_len, \ + void *value, const int value_len, const bool needLock) +{ + unsigned int hash_code; + HashData **ppBucket; + HashData *hash_data; + HashData *previous; + char *pBuff; + int bytes; + int malloc_value_size; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + previous = NULL; + + if (needLock) + { + HASH_LOCK(pHash, ppBucket - pHash->buckets) + } + + hash_data = *ppBucket; + while (hash_data != NULL) + { + if (key_len == hash_data->key_len && \ + memcmp(key, hash_data->key, key_len) == 0) + { + break; + } + + previous = hash_data; + hash_data = hash_data->next; + } + + if (hash_data != NULL) //exists + { + if (!pHash->is_malloc_value) + { + hash_data->value_len = value_len; + hash_data->value = (char *)value; + if (needLock) + { + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + } + return 0; + } + else + { + if (hash_data->malloc_value_size >= value_len && \ + (hash_data->malloc_value_size <= 128 || + hash_data->malloc_value_size / 2 < value_len)) + { + hash_data->value_len = value_len; + memcpy(hash_data->value, value, value_len); + if (needLock) + { + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + } + return 0; + } + + DELETE_FROM_BUCKET(pHash, ppBucket, previous, hash_data) + } + } + if (needLock) + { + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + } + + if (!pHash->is_malloc_value) + { + malloc_value_size = 0; + } + else + { + malloc_value_size = MEM_ALIGN(value_len); + } + + bytes = CALC_NODE_MALLOC_BYTES(key_len, malloc_value_size); + if (pHash->max_bytes > 0 && pHash->bytes_used+bytes > pHash->max_bytes) + { + return -ENOSPC; + } + + pBuff = (char *)malloc(bytes); + if (pBuff == NULL) + { + return -ENOMEM; + } + + pHash->bytes_used += bytes; + + hash_data = (HashData *)pBuff; + hash_data->malloc_value_size = malloc_value_size; + + hash_data->key_len = key_len; + memcpy(hash_data->key, key, key_len); +#ifdef HASH_STORE_HASH_CODE + hash_data->hash_code = hash_code; +#endif + hash_data->value_len = value_len; + + if (!pHash->is_malloc_value) + { + hash_data->value = (char *)value; + } + else + { + hash_data->value = hash_data->key + hash_data->key_len; + memcpy(hash_data->value, value, value_len); + } + + if (needLock) + { + HASH_LOCK(pHash, ppBucket - pHash->buckets) + ADD_TO_BUCKET(pHash, ppBucket, hash_data) + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + } + else + { + ADD_TO_BUCKET(pHash, ppBucket, hash_data) + } + + if (pHash->load_factor >= 0.10 && (double)pHash->item_count / + (double)*pHash->capacity >= pHash->load_factor) + { + _rehash(pHash); + } + + return 1; +} + +int64_t hash_inc_value(const HashData *old_data, const int inc, + char *new_value, int *new_value_len, void *arg) +{ + int64_t n; + if (old_data != NULL) + { + if (old_data->value_len < *new_value_len) + { + memcpy(new_value, old_data->value, old_data->value_len); + new_value[old_data->value_len] = '\0'; + n = strtoll(new_value, NULL, 10); + n += inc; + } + else + { + n = inc; + } + *new_value_len = sprintf(new_value, INT64_PRINTF_FORMAT, n); + } + else + { + n = inc; + *new_value_len = sprintf(new_value, INT64_PRINTF_FORMAT, n); + } + + return n; +} + +int hash_inc_ex(HashArray *pHash, const void *key, const int key_len, + const int inc, char *value, int *value_len, + ConvertValueFunc convert_func, void *arg) +{ + unsigned int hash_code; + int result; + HashData **ppBucket; + HashData *hash_data; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + HASH_LOCK(pHash, ppBucket - pHash->buckets) + hash_data = _chain_find_entry(ppBucket, key, key_len, hash_code); + convert_func(hash_data, inc, value, value_len, arg); + if (hash_data != NULL) + { + if (!pHash->is_malloc_value) + { + hash_data->value_len = *value_len; + hash_data->value = (char *)value; + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + return 0; + } + else + { + if (hash_data->malloc_value_size >= *value_len) + { + hash_data->value_len = *value_len; + memcpy(hash_data->value, value, *value_len); + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + return 0; + } + } + } + result = hash_insert_ex(pHash, key, key_len, value, *value_len, false); + if (result < 0) + { + *value = '\0'; + *value_len = 0; + result *= -1; + } + else + { + result = 0; + } + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + + return result; +} + +int hash_partial_set(HashArray *pHash, const void *key, const int key_len, + const char *value, const int offset, const int value_len) +{ + unsigned int hash_code; + int result; + HashData **ppBucket; + HashData *hash_data; + char *pNewBuff; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + HASH_LOCK(pHash, ppBucket - pHash->buckets) + hash_data = _chain_find_entry(ppBucket, key, key_len, hash_code); + do + { + if (hash_data != NULL) + { + if (offset < 0 || offset >= hash_data->value_len) + { + result = EINVAL; + break; + } + if (offset + value_len <= hash_data->value_len) + { + memcpy(hash_data->value+offset, value, value_len); + result = 0; + break; + } + + pNewBuff = (char *)malloc(offset + value_len); + if (pNewBuff == NULL) + { + result = errno != 0 ? errno : ENOMEM; + break; + } + + if (offset > 0) + { + memcpy(pNewBuff, hash_data->value, offset); + } + memcpy(pNewBuff + offset, value, value_len); + result = hash_insert_ex(pHash, key, key_len, pNewBuff, + offset + value_len, false); + free(pNewBuff); + } + else + { + if (offset != 0) + { + result = ENOENT; + break; + } + result = hash_insert_ex(pHash, key, key_len, (void *)value, + value_len, false); + } + + if (result < 0) + { + result *= -1; + } + else + { + result = 0; + } + } while (0); + + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + return result; +} + +int hash_delete(HashArray *pHash, const void *key, const int key_len) +{ + HashData **ppBucket; + HashData *hash_data; + HashData *previous; + unsigned int hash_code; + int result; + + hash_code = pHash->hash_func(key, key_len); + ppBucket = pHash->buckets + (hash_code % (*pHash->capacity)); + + result = ENOENT; + previous = NULL; + HASH_LOCK(pHash, ppBucket - pHash->buckets) + hash_data = *ppBucket; + while (hash_data != NULL) + { + if (key_len == hash_data->key_len && \ + memcmp(key, hash_data->key, key_len) == 0) + { + DELETE_FROM_BUCKET(pHash, ppBucket, previous, hash_data) + result = 0; + break; + } + + previous = hash_data; + hash_data = hash_data->next; + } + HASH_UNLOCK(pHash, ppBucket - pHash->buckets) + + return result; +} + +int hash_walk(HashArray *pHash, HashWalkFunc walkFunc, void *args) +{ + HashData **ppBucket; + HashData **bucket_end; + HashData *hash_data; + int index; + int result; + + index = 0; + bucket_end = pHash->buckets + (*pHash->capacity); + for (ppBucket=pHash->buckets; ppBucketnext; + } + } + + return 0; +} + +int hash_count(HashArray *pHash) +{ + return pHash->item_count; +} + +int hash_bucket_lock(HashArray *pHash, const unsigned int bucket_index) +{ + if (pHash->lock_count <= 0) + { + return 0; + } + + return pthread_mutex_lock(pHash->locks + bucket_index % + pHash->lock_count); +} + +int hash_bucket_unlock(HashArray *pHash, const unsigned int bucket_index) +{ + if (pHash->lock_count <= 0) + { + return 0; + } + + return pthread_mutex_unlock(pHash->locks + bucket_index % + pHash->lock_count); +} + +// RS Hash Function +int RSHash(const void *key, const int key_len) +{ + unsigned char *pKey; + unsigned char *pEnd; + int a = 63689; + int hash = 0; + + pEnd = (unsigned char *)key + key_len; + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) + { + hash = hash * a + (*pKey); + a *= 378551; + } + + return hash; +} + +#define JS_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int hash; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash ^= ((hash << 5) + (*pKey) + (hash >> 2)); \ + } \ + \ + return hash; \ + + +// JS Hash Function +int JSHash(const void *key, const int key_len) +{ + JS_HASH_FUNC(1315423911) +} + +int JSHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + JS_HASH_FUNC(init_value) +} + +#define BITS_IN_UNIGNED_INT (int)(sizeof(int) * 8) +#define THREE_QUARTERS (int)((BITS_IN_UNIGNED_INT * 3) / 4) +#define HASH_ONE_EIGHTH (int)(BITS_IN_UNIGNED_INT / 8) +#define HASH_HIGH_BITS (int)((unsigned int)(0xFFFFFFFF) << \ + (BITS_IN_UNIGNED_INT - HASH_ONE_EIGHTH)) + +#define PJW_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int hash; \ + int test; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash = (hash << HASH_ONE_EIGHTH) + (*(pKey)); \ + if ((test = hash & HASH_HIGH_BITS) != 0) \ + { \ + hash = ((hash ^ (test >> THREE_QUARTERS)) & (~HASH_HIGH_BITS)); \ + } \ + } \ + \ + return hash; \ + + +// P.J.Weinberger Hash Function, same as ELF Hash +int PJWHash(const void *key, const int key_len) +{ + PJW_HASH_FUNC(0) +} + +int PJWHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + PJW_HASH_FUNC(init_value) +} + +#define ELF_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int hash; \ + int x; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash = (hash << 4) + (*pKey); \ + if ((x = hash & 0xF0000000) != 0) \ + { \ + hash ^= (x >> 24); \ + hash &= ~x; \ + } \ + } \ + \ + return hash; \ + + +// ELF Hash Function, same as PJW Hash +int ELFHash(const void *key, const int key_len) +{ + ELF_HASH_FUNC(0) +} + +int ELFHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + ELF_HASH_FUNC(init_value) +} + +#define BKDR_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int seed = 131; /* 31 131 1313 13131 131313 etc..*/ \ + int hash; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash = hash * seed + (*pKey); \ + } \ + \ + return hash; \ + + +// BKDR Hash Function +int BKDRHash(const void *key, const int key_len) +{ + BKDR_HASH_FUNC(0) +} + +int BKDRHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + BKDR_HASH_FUNC(init_value) +} + +#define SDBM_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int hash; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash = (*pKey) + (hash << 6) + (hash << 16) - hash; \ + } \ + \ + return hash; \ + + +// SDBM Hash Function +int SDBMHash(const void *key, const int key_len) +{ + SDBM_HASH_FUNC(0) +} + +int SDBMHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + SDBM_HASH_FUNC(init_value) +} + +#define TIME33_HASH_FUNC(init_value) \ + int nHash; \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + \ + nHash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + nHash += (nHash << 5) + (*pKey); \ + } \ + \ + return nHash; \ + + +int Time33Hash(const void *key, const int key_len) +{ + TIME33_HASH_FUNC(0) +} + +int Time33Hash_ex(const void *key, const int key_len, \ + const int init_value) +{ + TIME33_HASH_FUNC(init_value) +} + +#define DJB_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int hash; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash += (hash << 5) + (*pKey); \ + } \ + \ + return hash; \ + + +// DJB Hash Function +int DJBHash(const void *key, const int key_len) +{ + DJB_HASH_FUNC(5381) +} + +int DJBHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + DJB_HASH_FUNC(init_value) +} + +#define AP_HASH_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int i; \ + int hash; \ + \ + hash = init_value; \ + \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key, i=0; pKey != pEnd; pKey++, i++) \ + { \ + if ((i & 1) == 0) \ + { \ + hash ^= ((hash << 7) ^ (*pKey) ^ (hash >> 3)); \ + } \ + else \ + { \ + hash ^= (~((hash << 11) ^ (*pKey) ^ (hash >> 5))); \ + } \ + } \ + \ + return hash; \ + + +// AP Hash Function +int APHash(const void *key, const int key_len) +{ + AP_HASH_FUNC(0) +} + +int APHash_ex(const void *key, const int key_len, \ + const int init_value) +{ + AP_HASH_FUNC(init_value) +} + +int calc_hashnr (const void* key, const int key_len) +{ + unsigned char *pKey; + unsigned char *pEnd; + int nr = 1, nr2 = 4; + + pEnd = (unsigned char *)key + key_len; + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) + { + nr ^= (((nr & 63) + nr2) * (*pKey)) + (nr << 8); + nr2 += 3; + } + + return nr; + +} + +#define CALC_HASHNR1_FUNC(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int hash; \ + \ + hash = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + hash *= 16777619; \ + hash ^= *pKey; \ + } \ + return hash; \ + +int calc_hashnr1(const void* key, const int key_len) +{ + CALC_HASHNR1_FUNC(0) +} + +int calc_hashnr1_ex(const void* key, const int key_len, \ + const int init_value) +{ + CALC_HASHNR1_FUNC(init_value) +} + +#define SIMPLE_HASH_FUNC(init_value) \ + int h; \ + unsigned char *p; \ + unsigned char *pEnd; \ + \ + h = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (p = (unsigned char *)key; p!= pEnd; p++) \ + { \ + h = 31 * h + *p; \ + } \ + \ + return h; \ + +int simple_hash(const void* key, const int key_len) +{ + SIMPLE_HASH_FUNC(0) +} + +int simple_hash_ex(const void* key, const int key_len, \ + const int init_value) +{ + SIMPLE_HASH_FUNC(init_value) +} + +static unsigned int crc_table[256] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D +}; + +#define CRC32_BODY(init_value) \ + unsigned char *pKey; \ + unsigned char *pEnd; \ + int crc; \ + \ + crc = init_value; \ + pEnd = (unsigned char *)key + key_len; \ + for (pKey = (unsigned char *)key; pKey != pEnd; pKey++) \ + { \ + crc = crc_table[(crc ^ *pKey) & 0xFF] ^ (crc >> 8); \ + } \ + +int CRC32(void *key, const int key_len) +{ + CRC32_BODY(CRC32_XINIT) + + return crc ^ CRC32_XOROT; +} + +int CRC32_ex(void *key, const int key_len, \ + const int init_value) +{ + CRC32_BODY(init_value) + + return crc; +} + diff --git a/common/hash.h b/common/hash.h new file mode 100644 index 0000000..9be6001 --- /dev/null +++ b/common/hash.h @@ -0,0 +1,381 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef _HASH_H_ +#define _HASH_H_ + +#include +#include +#include "common_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define CRC32_XINIT 0xFFFFFFFF /* initial value */ +#define CRC32_XOROT 0xFFFFFFFF /* final xor value */ + +typedef int (*HashFunc) (const void *key, const int key_len); + +#ifdef HASH_STORE_HASH_CODE +#define HASH_CODE(pHash, hash_data) hash_data->hash_code +#else +#define HASH_CODE(pHash, hash_data) ((unsigned int)pHash->hash_func( \ + hash_data->key, hash_data->key_len)) +#endif + +#define CALC_NODE_MALLOC_BYTES(key_len, value_size) \ + sizeof(HashData) + key_len + value_size + +#define FREE_HASH_DATA(pHash, hash_data) \ + pHash->item_count--; \ + pHash->bytes_used -= CALC_NODE_MALLOC_BYTES(hash_data->key_len, \ + hash_data->malloc_value_size); \ + free(hash_data); + + +typedef struct tagHashData +{ + int key_len; + int value_len; + int malloc_value_size; + +#ifdef HASH_STORE_HASH_CODE + unsigned int hash_code; +#endif + + char *value; + struct tagHashData *next; + char key[0]; +} HashData; + +typedef int64_t (*ConvertValueFunc)(const HashData *old_data, const int inc, + char *new_value, int *new_value_len, void *arg); + +typedef struct tagHashArray +{ + HashData **buckets; + HashFunc hash_func; + int item_count; + unsigned int *capacity; + double load_factor; + int64_t max_bytes; + int64_t bytes_used; + bool is_malloc_capacity; + bool is_malloc_value; + unsigned int lock_count; + pthread_mutex_t *locks; +} HashArray; + +typedef struct tagHashStat +{ + unsigned int capacity; + int item_count; + int bucket_used; + double bucket_avg_length; + int bucket_max_length; +} HashStat; + +/** + * hash walk function + * parameters: + * index: item index based 0 + * data: hash data, including key and value + * args: passed by hash_walk function + * return 0 for success, != 0 for error +*/ +typedef int (*HashWalkFunc)(const int index, const HashData *data, void *args); + +#define hash_init(pHash, hash_func, capacity, load_factor) \ + hash_init_ex(pHash, hash_func, capacity, load_factor, 0, false) + +#define hash_insert(pHash, key, key_len, value) \ + hash_insert_ex(pHash, key, key_len, value, 0, true) + +/** + * hash init function + * parameters: + * pHash: the hash table + * hash_func: hash function + * capacity: init capacity + * load_factor: hash load factor, such as 0.75 + * max_bytes: max memory can be used (bytes) + * bMallocValue: if need malloc value buffer + * return 0 for success, != 0 for error +*/ +int hash_init_ex(HashArray *pHash, HashFunc hash_func, \ + const unsigned int capacity, const double load_factor, \ + const int64_t max_bytes, const bool bMallocValue); + +/** + * set hash locks function + * parameters: + * lock_count: the lock count + * return 0 for success, != 0 for error +*/ +int hash_set_locks(HashArray *pHash, const int lock_count); + +/** + * convert the value + * parameters: + * HashData: the old hash data + * inc: the increasement value + * new_value: return the new value + * new_value_len: return the length of the new value + * arg: the user data + * return the number after increasement +*/ +int64_t hash_inc_value(const HashData *old_data, const int inc, + char *new_value, int *new_value_len, void *arg); + +#define hash_inc(pHash, key, key_len, inc, value, value_len) \ + hash_inc_ex(pHash, key, key_len, inc, value, value_len, \ + hash_inc_value, NULL) + +/** + * atomic increase value + * parameters: + * pHash: the hash table + * key: the key to insert + * key_len: length of th key + * inc: the increasement value + * value: return the new value + * value_len: return the length of the new value + * convert_func: the convert function + * arg: the arg to convert function + * return 0 for success, != 0 for error (errno) + * +*/ +int hash_inc_ex(HashArray *pHash, const void *key, const int key_len, + const int inc, char *value, int *value_len, + ConvertValueFunc convert_func, void *arg); + +/** + * hash destroy function + * parameters: + * pHash: the hash table + * return none +*/ +void hash_destroy(HashArray *pHash); + +/** + * hash insert key + * parameters: + * pHash: the hash table + * key: the key to insert + * key_len: length of th key + * value: the value + * value_len: length of the value + * needLock: if need lock + * return >= 0 for success, 0 for key already exist (update), + * 1 for new key (insert), < 0 for error +*/ +int hash_insert_ex(HashArray *pHash, const void *key, const int key_len, \ + void *value, const int value_len, const bool needLock); + +/** + * hash find key + * parameters: + * pHash: the hash table + * key: the key to find + * key_len: length of th key + * return user data, return NULL when the key not exist +*/ +void *hash_find(HashArray *pHash, const void *key, const int key_len); + +/** + * hash find key + * parameters: + * pHash: the hash table + * key: the key to find + * key_len: length of th key + * return hash data, return NULL when the key not exist +*/ +HashData *hash_find_ex(HashArray *pHash, const void *key, const int key_len); + + +/** + * hash get the value of the key + * parameters: + * pHash: the hash table + * key: the key to find + * key_len: length of th key + * value: store the value + * value_len: input for the max size of the value + * output for the length fo the value + * return 0 for success, != 0 fail (errno) +*/ +int hash_get(HashArray *pHash, const void *key, const int key_len, + void *value, int *value_len); + + +/** + * hash partial set + * parameters: + * pHash: the hash table + * key: the key to insert + * key_len: length of th key + * value: the value + * offset: the offset of existed value + * value_len: length of the value + * return 0 for success, != 0 fail (errno) +*/ +int hash_partial_set(HashArray *pHash, const void *key, const int key_len, + const char *value, const int offset, const int value_len); + +/** + * hash delete key + * parameters: + * pHash: the hash table + * key: the key to delete + * key_len: length of th key + * return 0 for success, != 0 fail (errno) +*/ +int hash_delete(HashArray *pHash, const void *key, const int key_len); + +/** + * hash walk (iterator) + * parameters: + * pHash: the hash table + * walkFunc: walk (interator) function + * args: extra args which will be passed to walkFunc + * return 0 for success, != 0 fail (errno) +*/ +int hash_walk(HashArray *pHash, HashWalkFunc walkFunc, void *args); + +/** + * get hash item count + * parameters: + * pHash: the hash table + * return item count +*/ +int hash_count(HashArray *pHash); + +/** + * hash best optimize + * parameters: + * pHash: the hash table + * suggest_capacity: suggest init capacity for speed + * return >0 for success, < 0 fail (errno) +*/ +int hash_best_op(HashArray *pHash, const int suggest_capacity); + +/** + * hash stat + * parameters: + * pHash: the hash table + * pStat: return stat info + * stat_by_lens: return stats array by bucket length + * stat_by_lens[0] empty buckets count + * stat_by_lens[1] contain 1 key buckets count + * stat_by_lens[2] contain 2 key buckets count, etc + * stat_size: stats array size (contain max elments) + * return 0 for success, != 0 fail (errno) +*/ +int hash_stat(HashArray *pHash, HashStat *pStat, \ + int *stat_by_lens, const int stat_size); + +/** + * print hash stat info + * parameters: + * pHash: the hash table + * return none +*/ +void hash_stat_print(HashArray *pHash); + +/** + * lock the bucket of hash table + * parameters: + * pHash: the hash table + * bucket_index: the index of bucket + * return 0 for success, != 0 fail (errno) +*/ +int hash_bucket_lock(HashArray *pHash, const unsigned int bucket_index); + +/** + * unlock the bucket of hash table + * parameters: + * pHash: the hash table + * bucket_index: the index of bucket + * return 0 for success, != 0 fail (errno) +*/ +int hash_bucket_unlock(HashArray *pHash, const unsigned int bucket_index); + +int RSHash(const void *key, const int key_len); + +int JSHash(const void *key, const int key_len); +int JSHash_ex(const void *key, const int key_len, \ + const int init_value); + +int PJWHash(const void *key, const int key_len); +int PJWHash_ex(const void *key, const int key_len, \ + const int init_value); + +int ELFHash(const void *key, const int key_len); +int ELFHash_ex(const void *key, const int key_len, \ + const int init_value); + +int BKDRHash(const void *key, const int key_len); +int BKDRHash_ex(const void *key, const int key_len, \ + const int init_value); + +int SDBMHash(const void *key, const int key_len); +int SDBMHash_ex(const void *key, const int key_len, \ + const int init_value); + +int Time33Hash(const void *key, const int key_len); +int Time33Hash_ex(const void *key, const int key_len, \ + const int init_value); + +int DJBHash(const void *key, const int key_len); +int DJBHash_ex(const void *key, const int key_len, \ + const int init_value); + +int APHash(const void *key, const int key_len); +int APHash_ex(const void *key, const int key_len, \ + const int init_value); + +int calc_hashnr (const void* key, const int key_len); + +int calc_hashnr1(const void* key, const int key_len); +int calc_hashnr1_ex(const void* key, const int key_len, \ + const int init_value); + +int simple_hash(const void* key, const int key_len); +int simple_hash_ex(const void* key, const int key_len, \ + const int init_value); + +int CRC32(void *key, const int key_len); +int CRC32_ex(void *key, const int key_len, \ + const int init_value); + +#define CRC32_FINAL(crc) (crc ^ CRC32_XOROT) + +#define INIT_HASH_CODES4(hash_codes) \ + hash_codes[0] = CRC32_XINIT; \ + hash_codes[1] = 0; \ + hash_codes[2] = 0; \ + hash_codes[3] = 0; \ + +#define CALC_HASH_CODES4(buff, buff_len, hash_codes) \ + hash_codes[0] = CRC32_ex(buff, buff_len, hash_codes[0]); \ + hash_codes[1] = ELFHash_ex(buff, buff_len, hash_codes[1]); \ + hash_codes[2] = simple_hash_ex(buff, buff_len, hash_codes[2]); \ + hash_codes[3] = Time33Hash_ex(buff, buff_len, hash_codes[3]); \ + + +#define FINISH_HASH_CODES4(hash_codes) \ + hash_codes[0] = CRC32_FINAL(hash_codes[0]); \ + + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/http_func.c b/common/http_func.c new file mode 100644 index 0000000..a543fc8 --- /dev/null +++ b/common/http_func.c @@ -0,0 +1,302 @@ + +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sockopt.h" +#include "logger.h" +#include "shared_func.h" + +int get_url_content(const char *url, const int connect_timeout, \ + const int network_timeout, int *http_status, \ + char **content, int *content_len, char *error_info) +{ + char domain_name[256]; + char ip_addr[IP_ADDRESS_SIZE]; + char out_buff[4096]; + int domain_len; + int url_len; + int out_len; + int alloc_size; + int recv_bytes; + int result; + int sock; + int port; + const char *pDomain; + const char *pContent; + const char *pURI; + char *pPort; + char *pSpace; + + *http_status = 0; + *content_len = 0; + *content = NULL; + + url_len = strlen(url); + if (url_len <= 7 || strncasecmp(url, "http://", 7) != 0) + { + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "invalid url.", __LINE__); + return EINVAL; + } + + pDomain = url + 7; + pURI = strchr(pDomain, '/'); + if (pURI == NULL) + { + domain_len = url_len - 7; + pURI = "/"; + } + else + { + domain_len = pURI - pDomain; + } + + if (domain_len >= sizeof(domain_name)) + { + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "domain is too large, exceed %d.", \ + __LINE__, (int)sizeof(domain_name)); + return EINVAL; + } + + memcpy(domain_name, pDomain, domain_len); + *(domain_name + domain_len) = '\0'; + pPort = strchr(domain_name, ':'); + if (pPort == NULL) + { + port = 80; + } + else + { + *pPort = '\0'; + port = atoi(pPort + 1); + } + + if (getIpaddrByName(domain_name, ip_addr, \ + sizeof(ip_addr)) == INADDR_NONE) + { + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "resolve domain \"%s\" fail.", \ + __LINE__, domain_name); + return EINVAL; + } + + sock = socket(AF_INET, SOCK_STREAM, 0); + if(sock < 0) + { + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s", __LINE__, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + if ((result=connectserverbyip_nb_auto(sock, ip_addr, port, \ + connect_timeout)) != 0) + { + close(sock); + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "connect to %s:%d fail, errno: %d, " \ + "error info: %s", __LINE__, domain_name, \ + port, result, STRERROR(result)); + + return result; + } + + out_len = snprintf(out_buff, sizeof(out_buff), \ + "GET %s HTTP/1.0\r\n" \ + "Host: %s:%d\r\n" \ + "Connection: close\r\n" \ + "\r\n", pURI, domain_name, port); + if ((result=tcpsenddata(sock, out_buff, out_len, network_timeout)) != 0) + { + close(sock); + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "send data to %s:%d fail, errno: %d, " \ + "error info: %s", __LINE__, domain_name, \ + port, result, STRERROR(result)); + + return result; + } + + alloc_size = 64 * 1024; + *content = (char *)malloc(alloc_size + 1); + if (*content == NULL) + { + close(sock); + result = errno != 0 ? errno : ENOMEM; + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, " \ + "error info: %s", __LINE__, alloc_size + 1, \ + result, STRERROR(result)); + + return result; + } + + do + { + recv_bytes = alloc_size - *content_len; + if (recv_bytes <= 0) + { + alloc_size *= 2; + *content = (char *)realloc(*content, alloc_size + 1); + if (*content == NULL) + { + close(sock); + result = errno != 0 ? errno : ENOMEM; + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "realloc %d bytes fail, errno: %d, " \ + "error info: %s", __LINE__, \ + alloc_size + 1, \ + result, STRERROR(result)); + + return result; + } + + recv_bytes = alloc_size - *content_len; + } + + result = tcprecvdata_ex(sock, *content + *content_len, \ + recv_bytes, network_timeout, &recv_bytes); + + *content_len += recv_bytes; + } while (result == 0); + + if (result != ENOTCONN) + { + close(sock); + free(*content); + *content = NULL; + *content_len = 0; + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "recv data from %s:%d fail, errno: %d, " \ + "error info: %s", __LINE__, domain_name, \ + port, result, STRERROR(result)); + + return result; + } + + *(*content + *content_len) = '\0'; + pContent = strstr(*content, "\r\n\r\n"); + if (pContent == NULL) + { + close(sock); + free(*content); + *content = NULL; + *content_len = 0; + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "response data from %s:%d is invalid", \ + __LINE__, domain_name, port); + + return EINVAL; + } + + pContent += 4; + pSpace = strchr(*content, ' '); + if (pSpace == NULL || pSpace >= pContent) + { + close(sock); + free(*content); + *content = NULL; + *content_len = 0; + + sprintf(error_info, "file: "__FILE__", line: %d, " \ + "response data from %s:%d is invalid", \ + __LINE__, domain_name, port); + + return EINVAL; + } + + *http_status = atoi(pSpace + 1); + *content_len -= pContent - *content; + memcpy(*content, pContent, *content_len); + *(*content + *content_len) = '\0'; + + close(sock); + + *error_info = '\0'; + return 0; +} + +int http_parse_query(char *url, KeyValuePair *params, const int max_count) +{ + KeyValuePair *pCurrent; + KeyValuePair *pEnd; + char *pParamStart; + char *p; + char *pStrEnd; + int value_len; + + pParamStart = strchr(url, '?'); + if (pParamStart == NULL) + { + return 0; + } + + *pParamStart = '\0'; + + pEnd = params + max_count; + pCurrent = params; + p = pParamStart + 1; + while (p != NULL && *p != '\0') + { + if (pCurrent >= pEnd) + { + return pCurrent - params; + } + + pCurrent->key = p; + pStrEnd = strchr(p, '&'); + if (pStrEnd == NULL) + { + p = NULL; + } + else + { + *pStrEnd = '\0'; + p = pStrEnd + 1; + } + + pStrEnd = strchr(pCurrent->key, '='); + if (pStrEnd == NULL) + { + continue; + } + + *pStrEnd = '\0'; + pCurrent->value = pStrEnd + 1; + if (*pCurrent->key == '\0') + { + continue; + } + + urldecode(pCurrent->value, strlen(pCurrent->value), \ + pCurrent->value, &value_len); + pCurrent++; + } + + return pCurrent - params; +} + diff --git a/common/http_func.h b/common/http_func.h new file mode 100644 index 0000000..b61edbb --- /dev/null +++ b/common/http_func.h @@ -0,0 +1,54 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef _HTTP_FUNC_H +#define _HTTP_FUNC_H + +#include +#include +#include +#include +#include +#include "common_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** +get content from url +params: + url: the url to fetch, must start as: "http://" + connect_timeout: connect timeout (seconds) + network_timeout: network timeout (seconds) + http_status: return http status code, 200 for Ok + content: return the content (HTTP body only, not including HTTP header), + *content should be freed by caller + content_len: return content length (bytes) +return: 0 for success, != 0 for error +**/ +int get_url_content(const char *url, const int connect_timeout, \ + const int network_timeout, int *http_status, \ + char **content, int *content_len, char *error_info); + +/** +parse url +params: + url: the url to parse + params: params array to store param and it's value + max_count: max param count +return: param count +**/ +int http_parse_query(char *url, KeyValuePair *params, const int max_count); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/ini_file_reader.c b/common/ini_file_reader.c new file mode 100644 index 0000000..f163c74 --- /dev/null +++ b/common/ini_file_reader.c @@ -0,0 +1,784 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//ini_file_reader.c + +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "logger.h" +#include "http_func.h" +#include "ini_file_reader.h" + +#define _LINE_BUFFER_SIZE 512 +#define _ALLOC_ITEMS_ONCE 8 + +static int iniDoLoadFromFile(const char *szFilename, \ + IniContext *pContext); +static int iniDoLoadItemsFromBuffer(char *content, \ + IniContext *pContext); + +static int iniCompareByItemName(const void *p1, const void *p2) +{ + return strcmp(((IniItem *)p1)->name, ((IniItem *)p2)->name); +} + +static int iniInitContext(IniContext *pContext) +{ + int result; + + memset(pContext, 0, sizeof(IniContext)); + pContext->current_section = &pContext->global; + if ((result=hash_init(&pContext->sections, PJWHash, 10, 0.75)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "hash_init fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return result; +} + +static int iniSortHashData(const int index, const HashData *data, void *args) +{ + IniSection *pSection; + + pSection = (IniSection *)data->value; + if (pSection->count > 1) + { + qsort(pSection->items, pSection->count, \ + sizeof(IniItem), iniCompareByItemName); + } + + return 0; +} + +static void iniSortItems(IniContext *pContext) +{ + if (pContext->global.count > 1) + { + qsort(pContext->global.items, pContext->global.count, \ + sizeof(IniItem), iniCompareByItemName); + } + + hash_walk(&pContext->sections, iniSortHashData, NULL); +} + +int iniLoadFromFile(const char *szFilename, IniContext *pContext) +{ + int result; + int len; + char *pLast; + char full_filename[MAX_PATH_SIZE]; + + if ((result=iniInitContext(pContext)) != 0) + { + return result; + } + + if (strncasecmp(szFilename, "http://", 7) == 0) + { + *pContext->config_path = '\0'; + snprintf(full_filename, sizeof(full_filename),"%s",szFilename); + } + else + { + if (*szFilename == '/') + { + pLast = strrchr(szFilename, '/'); + len = pLast - szFilename; + if (len >= sizeof(pContext->config_path)) + { + logError("file: "__FILE__", line: %d, "\ + "the path of the config file: %s is " \ + "too long!", __LINE__, szFilename); + return ENOSPC; + } + + memcpy(pContext->config_path, szFilename, len); + *(pContext->config_path + len) = '\0'; + snprintf(full_filename, sizeof(full_filename), \ + "%s", szFilename); + } + else + { + memset(pContext->config_path, 0, \ + sizeof(pContext->config_path)); + if (getcwd(pContext->config_path, sizeof( \ + pContext->config_path)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "getcwd fail, errno: %d, " \ + "error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + len = strlen(pContext->config_path); + if (len > 0 && pContext->config_path[len - 1] == '/') + { + len--; + *(pContext->config_path + len) = '\0'; + } + + snprintf(full_filename, sizeof(full_filename), \ + "%s/%s", pContext->config_path, szFilename); + + pLast = strrchr(szFilename, '/'); + if (pLast != NULL) + { + int tail_len; + + tail_len = pLast - szFilename; + if (len + tail_len >= sizeof( \ + pContext->config_path)) + { + logError("file: "__FILE__", line: %d, "\ + "the path of the config " \ + "file: %s is too long!", \ + __LINE__, szFilename); + return ENOSPC; + } + + memcpy(pContext->config_path + len, \ + szFilename, tail_len); + len += tail_len; + *(pContext->config_path + len) = '\0'; + } + } + } + + result = iniDoLoadFromFile(full_filename, pContext); + if (result == 0) + { + iniSortItems(pContext); + } + else + { + iniFreeContext(pContext); + } + + return result; +} + +static int iniDoLoadFromFile(const char *szFilename, \ + IniContext *pContext) +{ + char *content; + int result; + int http_status; + int content_len; + int64_t file_size; + char error_info[512]; + + if (strncasecmp(szFilename, "http://", 7) == 0) + { + if ((result=get_url_content(szFilename, 10, 60, &http_status, \ + &content, &content_len, error_info)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "get_url_content fail, " \ + "url: %s, error info: %s", \ + __LINE__, szFilename, error_info); + return result; + } + + if (http_status != 200) + { + free(content); + logError("file: "__FILE__", line: %d, " \ + "HTTP status code: %d != 200, url: %s", \ + __LINE__, http_status, szFilename); + return EINVAL; + } + } + else + { + if ((result=getFileContent(szFilename, &content, \ + &file_size)) != 0) + { + return result; + } + } + + result = iniDoLoadItemsFromBuffer(content, pContext); + free(content); + + return result; +} + +int iniLoadFromBuffer(char *content, IniContext *pContext) +{ + int result; + + if ((result=iniInitContext(pContext)) != 0) + { + return result; + } + + result = iniDoLoadItemsFromBuffer(content, pContext); + if (result == 0) + { + iniSortItems(pContext); + } + else + { + iniFreeContext(pContext); + } + + return result; +} + +static int iniDoLoadItemsFromBuffer(char *content, IniContext *pContext) +{ + IniSection *pSection; + IniItem *pItem; + char *pLine; + char *pLastEnd; + char *pEqualChar; + char *pIncludeFilename; + char full_filename[MAX_PATH_SIZE]; + int nLineLen; + int nNameLen; + int nValueLen; + int result; + + result = 0; + pLastEnd = content - 1; + pSection = pContext->current_section; + if (pSection->count > 0) + { + pItem = pSection->items + pSection->count; + } + else + { + pItem = pSection->items; + } + + while (pLastEnd != NULL) + { + pLine = pLastEnd + 1; + pLastEnd = strchr(pLine, '\n'); + if (pLastEnd != NULL) + { + *pLastEnd = '\0'; + } + + if (*pLine == '#' && \ + strncasecmp(pLine+1, "include", 7) == 0 && \ + (*(pLine+8) == ' ' || *(pLine+8) == '\t')) + { + pIncludeFilename = strdup(pLine + 9); + if (pIncludeFilename == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "strdup %d bytes fail", __LINE__, \ + (int)strlen(pLine + 9) + 1); + result = errno != 0 ? errno : ENOMEM; + break; + } + + trim(pIncludeFilename); + if (strncasecmp(pIncludeFilename, "http://", 7) == 0) + { + snprintf(full_filename, sizeof(full_filename),\ + "%s", pIncludeFilename); + } + else + { + if (*pIncludeFilename == '/') + { + snprintf(full_filename, sizeof(full_filename), \ + "%s", pIncludeFilename); + } + else + { + snprintf(full_filename, sizeof(full_filename), \ + "%s/%s", pContext->config_path, \ + pIncludeFilename); + } + + if (!fileExists(full_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "include file \"%s\" not exists, " \ + "line: \"%s\"", __LINE__, \ + pIncludeFilename, pLine); + free(pIncludeFilename); + result = ENOENT; + break; + } + } + + result = iniDoLoadFromFile(full_filename, pContext); + if (result != 0) + { + free(pIncludeFilename); + break; + } + + pSection = pContext->current_section; + if (pSection->count > 0) + { + pItem = pSection->items + pSection->count; //must re-asign + } + else + { + pItem = pSection->items; + } + + free(pIncludeFilename); + continue; + } + + trim(pLine); + if (*pLine == '#' || *pLine == '\0') + { + continue; + } + + nLineLen = strlen(pLine); + if (*pLine == '[' && *(pLine + (nLineLen - 1)) == ']') //section + { + char *section_name; + int section_len; + + *(pLine + (nLineLen - 1)) = '\0'; + section_name = pLine + 1; //skip [ + + trim(section_name); + if (*section_name == '\0') //global section + { + pContext->current_section = &pContext->global; + pSection = pContext->current_section; + if (pSection->count > 0) + { + pItem = pSection->items + pSection->count; + } + else + { + pItem = pSection->items; + } + continue; + } + + section_len = strlen(section_name); + pSection = (IniSection *)hash_find(&pContext->sections,\ + section_name, section_len); + if (pSection == NULL) + { + pSection = (IniSection *)malloc(sizeof(IniSection)); + if (pSection == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, "\ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, \ + (int)sizeof(IniSection), \ + result, STRERROR(result)); + + break; + } + + memset(pSection, 0, sizeof(IniSection)); + result = hash_insert(&pContext->sections, \ + section_name, section_len, pSection); + if (result < 0) + { + result *= -1; + logError("file: "__FILE__", line: %d, "\ + "insert into hash table fail, "\ + "errno: %d, error info: %s", \ + __LINE__, result, \ + STRERROR(result)); + break; + } + else + { + result = 0; + } + } + + pContext->current_section = pSection; + if (pSection->count > 0) + { + pItem = pSection->items + pSection->count; + } + else + { + pItem = pSection->items; + } + continue; + } + + pEqualChar = strchr(pLine, '='); + if (pEqualChar == NULL) + { + continue; + } + + nNameLen = pEqualChar - pLine; + nValueLen = strlen(pLine) - (nNameLen + 1); + if (nNameLen > FAST_INI_ITEM_NAME_LEN) + { + nNameLen = FAST_INI_ITEM_NAME_LEN; + } + + if (nValueLen > FAST_INI_ITEM_VALUE_LEN) + { + nValueLen = FAST_INI_ITEM_VALUE_LEN; + } + + if (pSection->count >= pSection->alloc_count) + { + pSection->alloc_count += _ALLOC_ITEMS_ONCE; + pSection->items=(IniItem *)realloc(pSection->items, + sizeof(IniItem) * pSection->alloc_count); + if (pSection->items == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "realloc %d bytes fail", __LINE__, \ + (int)sizeof(IniItem) * \ + pSection->alloc_count); + result = errno != 0 ? errno : ENOMEM; + break; + } + + pItem = pSection->items + pSection->count; + memset(pItem, 0, sizeof(IniItem) * \ + (pSection->alloc_count - pSection->count)); + } + + memcpy(pItem->name, pLine, nNameLen); + memcpy(pItem->value, pEqualChar + 1, nValueLen); + + trim(pItem->name); + trim(pItem->value); + + pSection->count++; + pItem++; + } + + return result; +} + +static int iniFreeHashData(const int index, const HashData *data, void *args) +{ + IniSection *pSection; + + pSection = (IniSection *)data->value; + if (pSection == NULL) + { + return 0; + } + + if (pSection->items != NULL) + { + free(pSection->items); + memset(pSection, 0, sizeof(IniSection)); + } + + free(pSection); + ((HashData *)data)->value = NULL; + return 0; +} + +void iniFreeContext(IniContext *pContext) +{ + if (pContext == NULL) + { + return; + } + + if (pContext->global.items != NULL) + { + free(pContext->global.items); + memset(&pContext->global, 0, sizeof(IniSection)); + } + + hash_walk(&pContext->sections, iniFreeHashData, NULL); + hash_destroy(&pContext->sections); +} + + +#define INI_FIND_ITEM(szSectionName, szItemName, pContext, pSection, \ + targetItem, pItem, return_val) \ + if (szSectionName == NULL || *szSectionName == '\0') \ + { \ + pSection = &pContext->global; \ + } \ + else \ + { \ + pSection = (IniSection *)hash_find(&pContext->sections, \ + szSectionName, strlen(szSectionName)); \ + if (pSection == NULL) \ + { \ + return return_val; \ + } \ + } \ + \ + if (pSection->count <= 0) \ + { \ + return return_val; \ + } \ + \ + snprintf(targetItem.name, sizeof(targetItem.name), "%s", szItemName); \ + pItem = (IniItem *)bsearch(&targetItem, pSection->items, \ + pSection->count, sizeof(IniItem), iniCompareByItemName); + + +char *iniGetStrValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext) +{ + IniItem targetItem; + IniSection *pSection; + IniItem *pItem; + + INI_FIND_ITEM(szSectionName, szItemName, pContext, pSection, \ + targetItem, pItem, NULL) + + if (pItem == NULL) + { + return NULL; + } + else + { + return pItem->value; + } +} + +int64_t iniGetInt64Value(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const int64_t nDefaultValue) +{ + char *pValue; + + pValue = iniGetStrValue(szSectionName, szItemName, pContext); + if (pValue == NULL) + { + return nDefaultValue; + } + else + { + return strtoll(pValue, NULL, 10); + } +} + +int iniGetIntValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const int nDefaultValue) +{ + char *pValue; + + pValue = iniGetStrValue(szSectionName, szItemName, pContext); + if (pValue == NULL) + { + return nDefaultValue; + } + else + { + return atoi(pValue); + } +} + +double iniGetDoubleValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const double dbDefaultValue) +{ + char *pValue; + + pValue = iniGetStrValue(szSectionName, szItemName, pContext); + if (pValue == NULL) + { + return dbDefaultValue; + } + else + { + return strtod(pValue, NULL); + } +} + +bool iniGetBoolValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const bool bDefaultValue) +{ + char *pValue; + + pValue = iniGetStrValue(szSectionName, szItemName, pContext); + if (pValue == NULL) + { + return bDefaultValue; + } + else + { + return strcasecmp(pValue, "true") == 0 || + strcasecmp(pValue, "yes") == 0 || + strcasecmp(pValue, "on") == 0 || + strcmp(pValue, "1") == 0; + } +} + +int iniGetValues(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, char **szValues, const int max_values) +{ + IniItem targetItem; + IniSection *pSection; + IniItem *pFound; + IniItem *pItem; + IniItem *pItemEnd; + char **ppValues; + + if (max_values <= 0) + { + return 0; + } + + INI_FIND_ITEM(szSectionName, szItemName, pContext, pSection, \ + targetItem, pFound, 0) + if (pFound == NULL) + { + return 0; + } + + ppValues = szValues; + *ppValues++ = pFound->value; + for (pItem=pFound-1; pItem>=pSection->items; pItem--) + { + if (strcmp(pItem->name, szItemName) != 0) + { + break; + } + + if (ppValues - szValues < max_values) + { + *ppValues++ = pItem->value; + } + } + + pItemEnd = pSection->items + pSection->count; + for (pItem=pFound+1; pItemname, szItemName) != 0) + { + break; + } + + if (ppValues - szValues < max_values) + { + *ppValues++ = pItem->value; + } + } + + return ppValues - szValues; +} + +IniItem *iniGetValuesEx(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, int *nTargetCount) +{ + IniItem targetItem; + IniSection *pSection; + IniItem *pFound; + IniItem *pItem; + IniItem *pItemEnd; + IniItem *pItemStart; + + *nTargetCount = 0; + INI_FIND_ITEM(szSectionName, szItemName, pContext, pSection, \ + targetItem, pFound, NULL) + if (pFound == NULL) + { + return NULL; + } + + *nTargetCount = 1; + for (pItem=pFound-1; pItem>=pSection->items; pItem--) + { + if (strcmp(pItem->name, szItemName) != 0) + { + break; + } + + (*nTargetCount)++; + } + pItemStart = pFound - (*nTargetCount) + 1; + + pItemEnd = pSection->items + pSection->count; + for (pItem=pFound+1; pItemname, szItemName) != 0) + { + break; + } + + (*nTargetCount)++; + } + + return pItemStart; +} + +static int iniPrintHashData(const int index, const HashData *data, void *args) +{ + IniSection *pSection; + IniItem *pItem; + IniItem *pItemEnd; + char section_name[256]; + int section_len; + int i; + + pSection = (IniSection *)data->value; + if (pSection == NULL) + { + return 0; + } + + section_len = data->key_len; + if (section_len >= sizeof(section_name)) + { + section_len = sizeof(section_name) - 1; + } + + memcpy(section_name, data->key, section_len); + *(section_name + section_len) = '\0'; + + printf("section: %s, item count: %d\n", section_name, pSection->count); + if (pSection->count > 0) + { + i = 0; + pItemEnd = pSection->items + pSection->count; + for (pItem=pSection->items; pItemname, pItem->value); + } + } + printf("\n"); + + return 0; +} + +void iniPrintItems(IniContext *pContext) +{ + IniItem *pItem; + IniItem *pItemEnd; + int i; + + printf("global section, item count: %d\n", pContext->global.count); + if (pContext->global.count > 0) + { + i = 0; + pItemEnd = pContext->global.items + pContext->global.count; + for (pItem=pContext->global.items; pItemname, pItem->value); + } + } + printf("\n"); + + hash_walk(&pContext->sections, iniPrintHashData, NULL); +} + diff --git a/common/ini_file_reader.h b/common/ini_file_reader.h new file mode 100644 index 0000000..b3ba6b7 --- /dev/null +++ b/common/ini_file_reader.h @@ -0,0 +1,166 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//ini_file_reader.h +#ifndef INI_FILE_READER_H +#define INI_FILE_READER_H + +#include +#include +#include +#include "common_define.h" +#include "hash.h" + +#define FAST_INI_ITEM_NAME_LEN 64 +#define FAST_INI_ITEM_VALUE_LEN 256 + +typedef struct +{ + char name[FAST_INI_ITEM_NAME_LEN + 1]; + char value[FAST_INI_ITEM_VALUE_LEN + 1]; +} IniItem; + +typedef struct +{ + IniItem *items; + int count; //item count + int alloc_count; +} IniSection; + +typedef struct +{ + IniSection global; + HashArray sections; //key is session name, and value is IniSection + IniSection *current_section; //for load from ini file + char config_path[MAX_PATH_SIZE]; //save the config filepath +} IniContext; + +#ifdef __cplusplus +extern "C" { +#endif + +/** load ini items from file + * parameters: + * szFilename: the filename, can be an URL + * pContext: the ini context + * return: error no, 0 for success, != 0 fail +*/ +int iniLoadFromFile(const char *szFilename, IniContext *pContext); + +/** load ini items from string buffer + * parameters: + * content: the string buffer to parse + * pContext: the ini context + * return: error no, 0 for success, != 0 fail +*/ +int iniLoadFromBuffer(char *content, IniContext *pContext); + +/** free ini context + * parameters: + * pContext: the ini context + * return: none +*/ +void iniFreeContext(IniContext *pContext); + +/** get item string value + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * return: item value, return NULL when the item not exist +*/ +char *iniGetStrValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext); + +/** get item string value + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * szValues: string array to store the values + * max_values: max string array elements + * return: item value count +*/ +int iniGetValues(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, char **szValues, const int max_values); + +/** get item int value (32 bits) + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * nDefaultValue: the default value + * return: item value, return nDefaultValue when the item not exist +*/ +int iniGetIntValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const int nDefaultValue); + +/** get item string value array + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * nTargetCount: store the item value count + * return: item value array, return NULL when the item not exist +*/ +IniItem *iniGetValuesEx(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, int *nTargetCount); + +/** get item int64 value (64 bits) + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * nDefaultValue: the default value + * return: int64 value, return nDefaultValue when the item not exist +*/ +int64_t iniGetInt64Value(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const int64_t nDefaultValue); + +/** get item boolean value + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * bDefaultValue: the default value + * return: item boolean value, return bDefaultValue when the item not exist +*/ +bool iniGetBoolValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const bool bDefaultValue); + +/** get item double value + * parameters: + * szSectionName: the section name, NULL or empty string for + * global section + * szItemName: the item name + * pContext: the ini context + * dbDefaultValue: the default value + * return: item value, return dbDefaultValue when the item not exist +*/ +double iniGetDoubleValue(const char *szSectionName, const char *szItemName, \ + IniContext *pContext, const double dbDefaultValue); + +/** print all items + * parameters: + * pContext: the ini context + * return: none +*/ +void iniPrintItems(IniContext *pContext); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/ioevent.c b/common/ioevent.c new file mode 100644 index 0000000..9ff0658 --- /dev/null +++ b/common/ioevent.c @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include +#include "ioevent.h" + +#if IOEVENT_USE_KQUEUE +/* we define these here as numbers, because for kqueue mapping them to a combination of + * filters / flags is hard to do. */ +int kqueue_ev_convert(int16_t event, uint16_t flags) +{ + int r; + + if (event == EVFILT_READ) { + r = KPOLLIN; + } + else if (event == EVFILT_WRITE) { + r = KPOLLOUT; + } + else { + r = 0; + } + + if (flags & EV_EOF) { + r |= KPOLLHUP; + } + return r; +} +#endif + +int ioevent_init(IOEventPoller *ioevent, const int size, + const int timeout, const int extra_events) +{ + int bytes; + + ioevent->size = size; + ioevent->extra_events = extra_events; + +#if IOEVENT_USE_EPOLL + ioevent->timeout = timeout; + ioevent->poll_fd = epoll_create(ioevent->size); + bytes = sizeof(struct epoll_event) * size; + ioevent->events = (struct epoll_event *)malloc(bytes); +#elif IOEVENT_USE_KQUEUE + ioevent->timeout.tv_sec = timeout / 1000; + ioevent->timeout.tv_nsec = 1000000 * (timeout % 1000); + ioevent->poll_fd = kqueue(); + bytes = sizeof(struct kevent) * size; + ioevent->events = (struct kevent *)malloc(bytes); +#elif IOEVENT_USE_PORT + ioevent->timeout.tv_sec = timeout / 1000; + ioevent->timeout.tv_nsec = 1000000 * (timeout % 1000); + ioevent->poll_fd = port_create(); + bytes = sizeof(port_event_t) * size; + ioevent->events = (port_event_t *)malloc(bytes); +#endif + if (ioevent->events == NULL) { + return errno != 0 ? errno : ENOMEM; + } + return 0; +} + +void ioevent_destroy(IOEventPoller *ioevent) +{ + if (ioevent->events != NULL) { + free(ioevent->events); + ioevent->events = NULL; + } + + if (ioevent->poll_fd >=0) { + close(ioevent->poll_fd); + ioevent->poll_fd = -1; + } +} + +int ioevent_attach(IOEventPoller *ioevent, const int fd, const int e, + void *data) +{ +#if IOEVENT_USE_EPOLL + struct epoll_event ev; + memset(&ev, 0, sizeof(ev)); + ev.events = e | ioevent->extra_events; + ev.data.ptr = data; + return epoll_ctl(ioevent->poll_fd, EPOLL_CTL_ADD, fd, &ev); +#elif IOEVENT_USE_KQUEUE + struct kevent ev[2]; + int n = 0; + if (e & IOEVENT_READ) { + EV_SET(&ev[n++], fd, EVFILT_READ, EV_ADD | ioevent->extra_events, 0, 0, data); + } + if (e & IOEVENT_WRITE) { + EV_SET(&ev[n++], fd, EVFILT_WRITE, EV_ADD | ioevent->extra_events, 0, 0, data); + } + return kevent(ioevent->poll_fd, ev, n, NULL, 0, NULL); +#elif IOEVENT_USE_PORT + return port_associate(ioevent->poll_fd, PORT_SOURCE_FD, fd, e, data); +#endif +} + +int ioevent_modify(IOEventPoller *ioevent, const int fd, const int e, + void *data) +{ +#if IOEVENT_USE_EPOLL + struct epoll_event ev; + memset(&ev, 0, sizeof(ev)); + ev.events = e | ioevent->extra_events; + ev.data.ptr = data; + return epoll_ctl(ioevent->poll_fd, EPOLL_CTL_MOD, fd, &ev); +#elif IOEVENT_USE_KQUEUE + struct kevent ev[2]; + int n = 0; + if (e & IOEVENT_READ) { + EV_SET(&ev[n++], fd, EVFILT_READ, EV_ADD | ioevent->extra_events, 0, 0, data); + } + else { + EV_SET(&ev[n++], fd, EVFILT_READ, EV_DELETE, 0, 0, data); + } + + if (e & IOEVENT_WRITE) { + EV_SET(&ev[n++], fd, EVFILT_WRITE, EV_ADD | ioevent->extra_events, 0, 0, data); + } + else { + EV_SET(&ev[n++], fd, EVFILT_WRITE, EV_DELETE, 0, 0, data); + } + return kevent(ioevent->poll_fd, ev, n, NULL, 0, NULL); +#elif IOEVENT_USE_PORT + return port_associate(ioevent->poll_fd, PORT_SOURCE_FD, fd, e, data); +#endif +} + +int ioevent_detach(IOEventPoller *ioevent, const int fd) +{ +#if IOEVENT_USE_EPOLL + return epoll_ctl(ioevent->poll_fd, EPOLL_CTL_DEL, fd, NULL); +#elif IOEVENT_USE_PORT + return port_dissociate(ioevent->poll_fd, PORT_SOURCE_FD, fd); +#else + return 0; +#endif +} + +int ioevent_poll(IOEventPoller *ioevent) +{ +#if IOEVENT_USE_EPOLL + return epoll_wait(ioevent->poll_fd, ioevent->events, ioevent->size, ioevent->timeout); +#elif IOEVENT_USE_KQUEUE + return kevent(ioevent->poll_fd, NULL, 0, ioevent->events, ioevent->size, &ioevent->timeout); +#elif IOEVENT_USE_PORT + int result; + int retval; + unsigned int nget = 1; + if((retval = port_getn(ioevent->poll_fd, ioevent->events, + ioevent->size, &nget, &ioevent->timeout)) == 0) + { + result = (int)nget; + } else { + switch(errno) { + case EINTR: + case EAGAIN: + case ETIME: + if (nget > 0) { + result = (int)nget; + } + else { + result = 0; + } + break; + default: + result = -1; + break; + } + } + return result; +#else +#error port me +#endif +} + diff --git a/common/ioevent.h b/common/ioevent.h new file mode 100644 index 0000000..79e2c5b --- /dev/null +++ b/common/ioevent.h @@ -0,0 +1,113 @@ +#ifndef __IOEVENT_H__ +#define __IOEVENT_H__ + +#include +#include +#include + +#define IOEVENT_TIMEOUT 0x8000 + +#if IOEVENT_USE_EPOLL +#include +#define IOEVENT_EDGE_TRIGGER EPOLLET + +#define IOEVENT_READ EPOLLIN +#define IOEVENT_WRITE EPOLLOUT +#define IOEVENT_ERROR (EPOLLERR | EPOLLPRI | EPOLLHUP) + +#elif IOEVENT_USE_KQUEUE +#include +#define IOEVENT_EDGE_TRIGGER EV_CLEAR + +#define KPOLLIN 0x001 +#define KPOLLPRI 0x002 +#define KPOLLOUT 0x004 +#define KPOLLERR 0x010 +#define KPOLLHUP 0x020 +#define IOEVENT_READ KPOLLIN +#define IOEVENT_WRITE KPOLLOUT +#define IOEVENT_ERROR (KPOLLHUP | KPOLLPRI | KPOLLHUP) + +#ifdef __cplusplus +extern "C" { +#endif + +int kqueue_ev_convert(int16_t event, uint16_t flags); + +#ifdef __cplusplus +} +#endif + +#elif IOEVENT_USE_PORT +#include +#define IOEVENT_EDGE_TRIGGER 0 + +#define IOEVENT_READ POLLIN +#define IOEVENT_WRITE POLLOUT +#define IOEVENT_ERROR (POLLERR | POLLPRI | POLLHUP) +#endif + +typedef struct ioevent_puller { + int size; //max events (fd) + int extra_events; + int poll_fd; + +#if IOEVENT_USE_EPOLL + struct epoll_event *events; + int timeout; +#elif IOEVENT_USE_KQUEUE + struct kevent *events; + struct timespec timeout; +#elif IOEVENT_USE_PORT + port_event_t *events; + timespec_t timeout; +#endif +} IOEventPoller; + +#if IOEVENT_USE_EPOLL + #define IOEVENT_GET_EVENTS(ioevent, index) \ + ioevent->events[index].events +#elif IOEVENT_USE_KQUEUE + #define IOEVENT_GET_EVENTS(ioevent, index) kqueue_ev_convert( \ + ioevent->events[index].filter, ioevent->events[index].flags) +#elif IOEVENT_USE_PORT + #define IOEVENT_GET_EVENTS(ioevent, index) \ + ioevent->events[index].portev_events +#else +#error port me +#endif + +#if IOEVENT_USE_EPOLL + #define IOEVENT_GET_DATA(ioevent, index) \ + ioevent->events[index].data.ptr +#elif IOEVENT_USE_KQUEUE + #define IOEVENT_GET_DATA(ioevent, index) \ + ioevent->events[index].udata +#elif IOEVENT_USE_PORT + #define IOEVENT_GET_DATA(ioevent, index) \ + ioevent->events[index].portev_user +#else +#error port me +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +int ioevent_init(IOEventPoller *ioevent, const int size, + const int timeout, const int extra_events); +void ioevent_destroy(IOEventPoller *ioevent); + +int ioevent_attach(IOEventPoller *ioevent, const int fd, const int e, + void *data); +int ioevent_modify(IOEventPoller *ioevent, const int fd, const int e, + void *data); +int ioevent_detach(IOEventPoller *ioevent, const int fd); +int ioevent_poll(IOEventPoller *ioevent); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/ioevent_loop.c b/common/ioevent_loop.c new file mode 100644 index 0000000..f2e2938 --- /dev/null +++ b/common/ioevent_loop.c @@ -0,0 +1,150 @@ +#include "sched_thread.h" +#include "logger.h" +#include "ioevent_loop.h" + +static void deal_ioevents(IOEventPoller *ioevent, const int count) +{ + int i; + int event; + IOEventEntry *pEntry; + for (i=0; icallback(pEntry->fd, event, pEntry->timer.data); + } +} + +static void deal_timeouts(FastTimerEntry *head) +{ + FastTimerEntry *entry; + FastTimerEntry *curent; + IOEventEntry *pEventEntry; + + entry = head->next; + while (entry != NULL) + { + curent = entry; + entry = entry->next; + + pEventEntry = (IOEventEntry *)curent->data; + if (pEventEntry != NULL) + { + pEventEntry->callback(pEventEntry->fd, IOEVENT_TIMEOUT, + curent->data); + } + } +} + +int ioevent_loop(struct nio_thread_data *pThreadData, + IOEventCallback recv_notify_callback, TaskCleanUpCallBack + clean_up_callback, volatile bool *continue_flag) +{ + int result; + IOEventEntry ev_notify; + FastTimerEntry head; + struct fast_task_info *pTask; + time_t last_check_time; + int count; + + memset(&ev_notify, 0, sizeof(ev_notify)); + ev_notify.fd = pThreadData->pipe_fds[0]; + ev_notify.callback = recv_notify_callback; + if (ioevent_attach(&pThreadData->ev_puller, + pThreadData->pipe_fds[0], IOEVENT_READ, + &ev_notify) != 0) + { + result = errno != 0 ? errno : ENOMEM; + logCrit("file: "__FILE__", line: %d, " \ + "ioevent_attach fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + last_check_time = g_current_time; + while (*continue_flag) + { + pThreadData->deleted_list = NULL; + count = ioevent_poll(&pThreadData->ev_puller); + if (count > 0) + { + deal_ioevents(&pThreadData->ev_puller, count); + } + else if (count < 0) + { + result = errno != 0 ? errno : EINVAL; + if (result != EINTR) + { + logError("file: "__FILE__", line: %d, " \ + "ioevent_poll fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + if (pThreadData->deleted_list != NULL) + { + count = 0; + while (pThreadData->deleted_list != NULL) + { + pTask = pThreadData->deleted_list; + pThreadData->deleted_list = pTask->next; + + clean_up_callback(pTask); + count++; + } + logInfo("cleanup task count: %d", count); + } + + if (g_current_time - last_check_time > 0) + { + last_check_time = g_current_time; + count = fast_timer_timeouts_get( + &pThreadData->timer, g_current_time, &head); + if (count > 0) + { + deal_timeouts(&head); + } + } + } + + return 0; +} + +int ioevent_set(struct fast_task_info *pTask, struct nio_thread_data *pThread, + int sock, short event, IOEventCallback callback, const int timeout) +{ + int result; + + pTask->thread_data = pThread; + pTask->event.fd = sock; + pTask->event.callback = callback; + if (ioevent_attach(&pThread->ev_puller, + sock, event, pTask) < 0) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "ioevent_attach fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + pTask->event.timer.data = pTask; + pTask->event.timer.expires = g_current_time + timeout; + result = fast_timer_add(&pThread->timer, &pTask->event.timer); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "fast_timer_add fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + diff --git a/common/ioevent_loop.h b/common/ioevent_loop.h new file mode 100644 index 0000000..dbed3a1 --- /dev/null +++ b/common/ioevent_loop.h @@ -0,0 +1,22 @@ +#ifndef _IOEVENT_LOOP_H +#define _IOEVENT_LOOP_H + +#include "fast_task_queue.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int ioevent_loop(struct nio_thread_data *pThreadData, + IOEventCallback recv_notify_callback, TaskCleanUpCallBack + clean_up_callback, volatile bool *continue_flag); + +int ioevent_set(struct fast_task_info *pTask, struct nio_thread_data *pThread, + int sock, short event, IOEventCallback callback, const int timeout); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/linux_stack_trace.c b/common/linux_stack_trace.c new file mode 100644 index 0000000..800e1a6 --- /dev/null +++ b/common/linux_stack_trace.c @@ -0,0 +1,163 @@ +/** + * This source file is used to print out a stack-trace when your program + * segfaults. It is relatively reliable and spot-on accurate. + * + * This code is in the public domain. Use it as you see fit, some credit + * would be appreciated, but is not a prerequisite for usage. Feedback + * on it's use would encourage further development and maintenance. + * + * Due to a bug in gcc-4.x.x you currently have to compile as C++ if you want + * demangling to work. + * + * Please note that it's been ported into my ULS library, thus the check for + * + * Author: Jaco Kroon + * + * Copyright (C) 2005 - 2009 Jaco Kroon + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +/* Bug in gcc prevents from using CPP_DEMANGLE in pure "C" */ +#if !defined(__cplusplus) && !defined(NO_CPP_DEMANGLE) +#define NO_CPP_DEMANGLE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "shared_func.h" +#ifndef NO_CPP_DEMANGLE +#include +#ifdef __cplusplus +using __cxxabiv1::__cxa_demangle; +#endif +#endif + +#if defined(REG_RIP) +# define SIGSEGV_STACK_IA64 +# define REGFORMAT "%016lx" +#elif defined(REG_EIP) +# define SIGSEGV_STACK_X86 +# define REGFORMAT "%08x" +#else +# define SIGSEGV_STACK_GENERIC +# define REGFORMAT "%x" +#endif + +extern char g_exe_name[256]; + +void signal_stack_trace_print(int signum, siginfo_t *info, void *ptr) +{ + static const char *si_codes[3] = {"", "SEGV_MAPERR", "SEGV_ACCERR"}; + + int i, f = 0; + ucontext_t *ucontext; + Dl_info dlinfo; + void **bp = NULL; + void *ip = NULL; + char cmd[256]; + char buff[256]; + char output[8 * 1024]; + char *pCurrent; + + pCurrent = output; + ucontext = (ucontext_t*)ptr; + pCurrent += sprintf(pCurrent, "Segmentation Fault!\n"); + pCurrent += sprintf(pCurrent, "\tinfo.si_signo = %d\n", signum); + pCurrent += sprintf(pCurrent, "\tinfo.si_errno = %d\n", info->si_errno); + pCurrent += sprintf(pCurrent, "\tinfo.si_code = %d (%s)\n", \ + info->si_code, si_codes[info->si_code]); + pCurrent += sprintf(pCurrent, "\tinfo.si_addr = %p\n", info->si_addr); + for(i = 0; i < NGREG; i++) + { + pCurrent += sprintf(pCurrent, "\treg[%02d] = 0x"REGFORMAT"\n", + i, ucontext->uc_mcontext.gregs[i]); + } + +#ifndef SIGSEGV_NOSTACK +#if defined(SIGSEGV_STACK_IA64) || defined(SIGSEGV_STACK_X86) +#if defined(SIGSEGV_STACK_IA64) + ip = (void*)ucontext->uc_mcontext.gregs[REG_RIP]; + bp = (void**)ucontext->uc_mcontext.gregs[REG_RBP]; +#elif defined(SIGSEGV_STACK_X86) + ip = (void*)ucontext->uc_mcontext.gregs[REG_EIP]; + bp = (void**)ucontext->uc_mcontext.gregs[REG_EBP]; +#endif + + pCurrent += sprintf(pCurrent, "\tStack trace:\n"); + while(bp && ip) + { + const char *symname; +#ifndef NO_CPP_DEMANGLE + int status; + char * tmp; +#endif + + if(!dladdr(ip, &dlinfo)) + { + break; + } + + symname = dlinfo.dli_sname; + +#ifndef NO_CPP_DEMANGLE + tmp = __cxa_demangle(symname, NULL, 0, &status); + + if (status == 0 && tmp) + { + symname = tmp; + } +#endif + + sprintf(cmd, "addr2line -e %s %p", g_exe_name, ip); + if (getExecResult(cmd, buff, sizeof(buff)) != 0) + { + *buff = '0'; + } + + pCurrent += sprintf(pCurrent, "\t\t% 2d: %p <%s+%lu> (%s in %s)\n", + ++f, ip, symname, + (unsigned long)ip-(unsigned long)dlinfo.dli_saddr, + trim_right(buff), dlinfo.dli_fname); + + +#ifndef NO_CPP_DEMANGLE + if (tmp) + { + free(tmp); + } +#endif + + if(dlinfo.dli_sname && !strcmp(dlinfo.dli_sname, "main")) + { + break; + } + + ip = bp[1]; + bp = (void**)bp[0]; + } +#else + pCurrent += sprintf(pCurrent, "\tStack trace (non-dedicated):\n"); + sz = backtrace(bt, 20); + strings = backtrace_symbols(bt, sz); + for(i = 0; i < sz; ++i) + { + pCurrent += sprintf(pCurrent, "\t\t%s\n", strings[i]); + } +#endif + pCurrent += sprintf(pCurrent, "\tEnd of stack trace.\n"); +#else + pCurrent += sprintf(pCurrent, "\tNot printing stack strace.\n"); +#endif + + log_it_ex(&g_log_context, LOG_CRIT, output, pCurrent - output); +} + diff --git a/common/linux_stack_trace.h b/common/linux_stack_trace.h new file mode 100644 index 0000000..9fce7e2 --- /dev/null +++ b/common/linux_stack_trace.h @@ -0,0 +1,26 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//linux_stack_trace.h +#ifndef LINUX_STACK_TRACE_H +#define LINUX_STACK_TRACE_H + +#include "common_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void signal_stack_trace_print(int signum, siginfo_t *info, void *ptr); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/local_ip_func.c b/common/local_ip_func.c new file mode 100644 index 0000000..d2829fb --- /dev/null +++ b/common/local_ip_func.c @@ -0,0 +1,134 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include "logger.h" +#include "sockopt.h" +#include "shared_func.h" +#include "local_ip_func.h" + +int g_local_host_ip_count = 0; +char g_local_host_ip_addrs[FAST_MAX_LOCAL_IP_ADDRS * \ + IP_ADDRESS_SIZE]; +char g_if_alias_prefix[FAST_IF_ALIAS_PREFIX_MAX_SIZE] = {0}; + +bool is_local_host_ip(const char *client_ip) +{ + char *p; + char *pEnd; + + pEnd = g_local_host_ip_addrs + \ + IP_ADDRESS_SIZE * g_local_host_ip_count; + for (p=g_local_host_ip_addrs; p= FAST_MAX_LOCAL_IP_ADDRS) + { + return -1; + } + + strcpy(g_local_host_ip_addrs + \ + IP_ADDRESS_SIZE * g_local_host_ip_count, \ + client_ip); + g_local_host_ip_count++; + return 1; +} + +static void log_local_host_ip_addrs() +{ + char *p; + char *pEnd; + char buff[512]; + int len; + + len = sprintf(buff, "local_host_ip_count: %d,", g_local_host_ip_count); + pEnd = g_local_host_ip_addrs + \ + IP_ADDRESS_SIZE * g_local_host_ip_count; + for (p=g_local_host_ip_addrs; p +#include +#include +#include +#include "common_define.h" + +#define FAST_IF_ALIAS_PREFIX_MAX_SIZE 32 +#define FAST_MAX_LOCAL_IP_ADDRS 16 + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_local_host_ip_count; +extern char g_local_host_ip_addrs[FAST_MAX_LOCAL_IP_ADDRS * \ + IP_ADDRESS_SIZE]; +extern char g_if_alias_prefix[FAST_IF_ALIAS_PREFIX_MAX_SIZE]; + +void load_local_host_ip_addrs(); +bool is_local_host_ip(const char *client_ip); +int insert_into_local_host_ip(const char *client_ip); +void print_local_host_ip_addrs(); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/logger.c b/common/logger.c new file mode 100644 index 0000000..f66a763 --- /dev/null +++ b/common/logger.c @@ -0,0 +1,668 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "logger.h" + +#ifndef LINE_MAX +#define LINE_MAX 2048 +#endif + +#define LOG_BUFF_SIZE 64 * 1024 + +LogContext g_log_context = {LOG_INFO, STDERR_FILENO, NULL}; + +static int log_fsync(LogContext *pContext, const bool bNeedLock); + +static int check_and_mk_log_dir(const char *base_path) +{ + char data_path[MAX_PATH_SIZE]; + + snprintf(data_path, sizeof(data_path), "%s/logs", base_path); + if (!fileExists(data_path)) + { + if (mkdir(data_path, 0755) != 0) + { + fprintf(stderr, "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + data_path, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + } + + return 0; +} + +int log_init() +{ + if (g_log_context.log_buff != NULL) + { + return 0; + } + + return log_init_ex(&g_log_context); +} + +int log_init_ex(LogContext *pContext) +{ + int result; + + memset(pContext, 0, sizeof(LogContext)); + pContext->log_level = LOG_INFO; + pContext->log_fd = STDERR_FILENO; + pContext->log_to_cache = false; + pContext->rotate_immediately = false; + pContext->time_precision = LOG_TIME_PRECISION_SECOND; + + pContext->log_buff = (char *)malloc(LOG_BUFF_SIZE); + if (pContext->log_buff == NULL) + { + fprintf(stderr, "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + LOG_BUFF_SIZE, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + pContext->pcurrent_buff = pContext->log_buff; + + if ((result=init_pthread_lock(&(pContext->log_thread_lock))) != 0) + { + return result; + } + + return 0; +} + +static int log_open(LogContext *pContext) +{ + if ((pContext->log_fd = open(pContext->log_filename, O_WRONLY | \ + O_CREAT | O_APPEND, 0644)) < 0) + { + fprintf(stderr, "open log file \"%s\" to write fail, " \ + "errno: %d, error info: %s", \ + pContext->log_filename, errno, STRERROR(errno)); + pContext->log_fd = STDERR_FILENO; + return errno != 0 ? errno : EACCES; + } + + pContext->current_size = lseek(pContext->log_fd, 0, SEEK_END); + if (pContext->current_size < 0) + { + fprintf(stderr, "lseek file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + pContext->log_filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +int log_set_prefix_ex(LogContext *pContext, const char *base_path, \ + const char *filename_prefix) +{ + int result; + + if ((result=check_and_mk_log_dir(base_path)) != 0) + { + return result; + } + + snprintf(pContext->log_filename, MAX_PATH_SIZE, "%s/logs/%s.log", \ + base_path, filename_prefix); + + return log_open(pContext); +} + +int log_set_filename_ex(LogContext *pContext, const char *log_filename) +{ + snprintf(pContext->log_filename, MAX_PATH_SIZE, "%s", log_filename); + return log_open(pContext); +} + +void log_set_cache_ex(LogContext *pContext, const bool bLogCache) +{ + pContext->log_to_cache = bLogCache; +} + +void log_set_time_precision(LogContext *pContext, const int time_precision) +{ + pContext->time_precision = time_precision; +} + +void log_destroy_ex(LogContext *pContext) +{ + if (pContext->log_fd >= 0 && pContext->log_fd != STDERR_FILENO) + { + log_fsync(pContext, true); + + close(pContext->log_fd); + pContext->log_fd = STDERR_FILENO; + + pthread_mutex_destroy(&pContext->log_thread_lock); + } + + if (pContext->log_buff != NULL) + { + free(pContext->log_buff); + pContext->log_buff = NULL; + pContext->pcurrent_buff = NULL; + } +} + +int log_sync_func(void *args) +{ + if (args == NULL) + { + return EINVAL; + } + + return log_fsync((LogContext *)args, true); +} + +int log_notify_rotate(void *args) +{ + if (args == NULL) + { + return EINVAL; + } + + ((LogContext *)args)->rotate_immediately = true; + return 0; +} + +static int log_rotate(LogContext *pContext) +{ + struct tm tm; + time_t current_time; + char new_filename[MAX_PATH_SIZE + 32]; + + if (*(pContext->log_filename) == '\0') + { + return ENOENT; + } + + close(pContext->log_fd); + + current_time = get_current_time(); + localtime_r(¤t_time, &tm); + sprintf(new_filename, "%s.%04d%02d%02d_%02d%02d%02d", \ + pContext->log_filename, \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec); + if (rename(pContext->log_filename, new_filename) != 0) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "rename %s to %s fail, errno: %d, error info: %s", \ + __LINE__, pContext->log_filename, new_filename, \ + errno, STRERROR(errno)); + } + + return log_open(pContext); +} + +static int log_check_rotate(LogContext *pContext, const bool bNeedLock) +{ + int result; + + if (pContext->log_fd == STDERR_FILENO) + { + if (pContext->current_size > 0) + { + pContext->current_size = 0; + } + return ENOENT; + } + + if (bNeedLock) + { + pthread_mutex_lock(&(pContext->log_thread_lock)); + } + + if (pContext->rotate_immediately) + { + result = log_rotate(pContext); + pContext->rotate_immediately = false; + } + else + { + result = 0; + } + + if (bNeedLock) + { + pthread_mutex_unlock(&(pContext->log_thread_lock)); + } + + return result; +} + +static int log_fsync(LogContext *pContext, const bool bNeedLock) +{ + int result; + int lock_res; + int write_bytes; + + write_bytes = pContext->pcurrent_buff - pContext->log_buff; + if (write_bytes == 0) + { + if (!pContext->rotate_immediately) + { + return 0; + } + else + { + return log_check_rotate(pContext, bNeedLock); + } + } + + if (bNeedLock && ((lock_res=pthread_mutex_lock( \ + &(pContext->log_thread_lock))) != 0)) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, lock_res, STRERROR(lock_res)); + } + + if (pContext->rotate_size > 0) + { + pContext->current_size += write_bytes; + if (pContext->current_size > pContext->rotate_size) + { + pContext->rotate_immediately = true; + log_check_rotate(pContext, false); + } + } + + result = 0; + do + { + write_bytes = pContext->pcurrent_buff - pContext->log_buff; + if (write(pContext->log_fd, pContext->log_buff, write_bytes) != \ + write_bytes) + { + result = errno != 0 ? errno : EIO; + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "call write fail, errno: %d, error info: %s\n",\ + __LINE__, result, STRERROR(result)); + break; + } + + if (pContext->log_fd != STDERR_FILENO) + { + if (fsync(pContext->log_fd) != 0) + { + result = errno != 0 ? errno : EIO; + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "call fsync fail, errno: %d, error info: %s\n",\ + __LINE__, result, STRERROR(result)); + break; + } + } + + if (pContext->rotate_immediately) + { + result = log_check_rotate(pContext, false); + } + } while (0); + + pContext->pcurrent_buff = pContext->log_buff; + if (bNeedLock && ((lock_res=pthread_mutex_unlock( \ + &(pContext->log_thread_lock))) != 0)) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, lock_res, STRERROR(lock_res)); + } + + return result; +} + +static void doLogEx(LogContext *pContext, struct timeval *tv, \ + const char *caption, const char *text, const int text_len, \ + const bool bNeedSync) +{ + struct tm tm; + int time_fragment; + int buff_len; + int result; + + if (pContext->time_precision == LOG_TIME_PRECISION_SECOND) + { + time_fragment = 0; + } + else + { + if (pContext->time_precision == LOG_TIME_PRECISION_MSECOND) + { + time_fragment = tv->tv_usec / 1000; + } + else + { + time_fragment = tv->tv_usec; + } + } + + localtime_r(&tv->tv_sec, &tm); + if ((result=pthread_mutex_lock(&pContext->log_thread_lock)) != 0) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if (text_len + 64 > LOG_BUFF_SIZE) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "log buff size: %d < log text length: %d ", \ + __LINE__, LOG_BUFF_SIZE, text_len + 64); + pthread_mutex_unlock(&(pContext->log_thread_lock)); + return; + } + + if ((pContext->pcurrent_buff - pContext->log_buff) + text_len + 64 \ + > LOG_BUFF_SIZE) + { + log_fsync(pContext, false); + } + + if (pContext->time_precision == LOG_TIME_PRECISION_SECOND) + { + buff_len = sprintf(pContext->pcurrent_buff, \ + "[%04d-%02d-%02d %02d:%02d:%02d] ", \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec); + } + else + { + buff_len = sprintf(pContext->pcurrent_buff, \ + "[%04d-%02d-%02d %02d:%02d:%02d.%03d] ", \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec, time_fragment); + } + pContext->pcurrent_buff += buff_len; + + if (caption != NULL) + { + buff_len = sprintf(pContext->pcurrent_buff, "%s - ", caption); + pContext->pcurrent_buff += buff_len; + } + memcpy(pContext->pcurrent_buff, text, text_len); + pContext->pcurrent_buff += text_len; + *pContext->pcurrent_buff++ = '\n'; + + if (!pContext->log_to_cache || bNeedSync) + { + log_fsync(pContext, false); + } + + if ((result=pthread_mutex_unlock(&(pContext->log_thread_lock))) != 0) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } +} + +static void doLog(LogContext *pContext, const char *caption, \ + const char *text, const int text_len, const bool bNeedSync) +{ + struct timeval tv; + + if (pContext->time_precision == LOG_TIME_PRECISION_SECOND) + { + tv.tv_sec = get_current_time(); + tv.tv_usec = 0; + } + else + { + gettimeofday(&tv, NULL); + } + + doLogEx(pContext, &tv, caption, text, text_len, bNeedSync); +} + +void log_it_ex1(LogContext *pContext, const int priority, \ + const char *text, const int text_len) +{ + bool bNeedSync; + char *caption; + + switch(priority) + { + case LOG_DEBUG: + bNeedSync = true; + caption = "DEBUG"; + break; + case LOG_INFO: + bNeedSync = true; + caption = "INFO"; + break; + case LOG_NOTICE: + bNeedSync = false; + caption = "NOTICE"; + break; + case LOG_WARNING: + bNeedSync = false; + caption = "WARNING"; + break; + case LOG_ERR: + bNeedSync = false; + caption = "ERROR"; + break; + case LOG_CRIT: + bNeedSync = true; + caption = "CRIT"; + break; + case LOG_ALERT: + bNeedSync = true; + caption = "ALERT"; + break; + case LOG_EMERG: + bNeedSync = true; + caption = "EMERG"; + break; + default: + bNeedSync = false; + caption = "UNKOWN"; + break; + } + + doLog(pContext, caption, text, text_len, bNeedSync); +} + +void log_it_ex(LogContext *pContext, const int priority, const char *format, ...) +{ + bool bNeedSync; + char text[LINE_MAX]; + char *caption; + int len; + + va_list ap; + va_start(ap, format); + len = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + + switch(priority) + { + case LOG_DEBUG: + bNeedSync = true; + caption = "DEBUG"; + break; + case LOG_INFO: + bNeedSync = true; + caption = "INFO"; + break; + case LOG_NOTICE: + bNeedSync = false; + caption = "NOTICE"; + break; + case LOG_WARNING: + bNeedSync = false; + caption = "WARNING"; + break; + case LOG_ERR: + bNeedSync = false; + caption = "ERROR"; + break; + case LOG_CRIT: + bNeedSync = true; + caption = "CRIT"; + break; + case LOG_ALERT: + bNeedSync = true; + caption = "ALERT"; + break; + case LOG_EMERG: + bNeedSync = true; + caption = "EMERG"; + break; + default: + bNeedSync = false; + caption = "UNKOWN"; + break; + } + + doLog(pContext, caption, text, len, bNeedSync); +} + + +#define _DO_LOG(pContext, priority, caption, bNeedSync) \ + char text[LINE_MAX]; \ + int len; \ +\ + if (pContext->log_level < priority) \ + { \ + return; \ + } \ +\ + { \ + va_list ap; \ + va_start(ap, format); \ + len = vsnprintf(text, sizeof(text), format, ap); \ + va_end(ap); \ + } \ +\ + doLog(pContext, caption, text, len, bNeedSync); \ + + +void logEmergEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_EMERG, "EMERG", true) +} + +void logAlertEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_ALERT, "ALERT", true) +} + +void logCritEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_CRIT, "CRIT", true) +} + +void logErrorEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_ERR, "ERROR", false) +} + +void logWarningEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_WARNING, "WARNING", false) +} + +void logNoticeEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_NOTICE, "NOTICE", false) +} + +void logInfoEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_INFO, "INFO", false) +} + +void logDebugEx(LogContext *pContext, const char *format, ...) +{ + _DO_LOG(pContext, LOG_DEBUG, "DEBUG", false) +} + +void logAccess(LogContext *pContext, struct timeval *tvStart, \ + const char *format, ...) +{ + char text[LINE_MAX]; + int len; + va_list ap; + + va_start(ap, format); + len = vsnprintf(text, sizeof(text), format, ap); + va_end(ap); + + doLogEx(pContext, tvStart, NULL, text, len, false); +} + +#ifndef LOG_FORMAT_CHECK + +void logEmerg(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_EMERG, "EMERG", true) +} + +void logAlert(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_ALERT, "ALERT", true) +} + +void logCrit(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_CRIT, "CRIT", true) +} + +void logError(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_ERR, "ERROR", false) +} + +void logWarning(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_WARNING, "WARNING", false) +} + +void logNotice(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_NOTICE, "NOTICE", false) +} + +void logInfo(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_INFO, "INFO", true) +} + +void logDebug(const char *format, ...) +{ + _DO_LOG((&g_log_context), LOG_DEBUG, "DEBUG", true) +} + +#endif + diff --git a/common/logger.h b/common/logger.h new file mode 100644 index 0000000..79f5025 --- /dev/null +++ b/common/logger.h @@ -0,0 +1,212 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//logger.h +#ifndef LOGGER_H +#define LOGGER_H + +#include +#include +#include "common_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_TIME_PRECISION_SECOND 's' //second +#define LOG_TIME_PRECISION_MSECOND 'm' //millisecond +#define LOG_TIME_PRECISION_USSECOND 'u' //microsecond + +typedef struct log_context +{ + /* log level value please see: sys/syslog.h + default value is LOG_INFO */ + int log_level; + + /* default value is STDERR_FILENO */ + int log_fd; + + /* cache buffer */ + char *log_buff; + + /* string end in the cache buffer for next sprintf */ + char *pcurrent_buff; + + /* mutext lock */ + pthread_mutex_t log_thread_lock; + + /* + rotate the log when the log file exceeds this parameter + rotate_size > 0 means need rotate log by log file size + */ + int64_t rotate_size; + + /* log file current size */ + int64_t current_size; + + /* if write to buffer firstly, then sync to disk. + default value is false (no cache) */ + bool log_to_cache; + + /* if rotate the access log */ + bool rotate_immediately; + + /* time precision */ + char time_precision; + + /* save the log filename */ + char log_filename[MAX_PATH_SIZE]; +} LogContext; + +extern LogContext g_log_context; + +/** init function using global log context + * return: 0 for success, != 0 fail +*/ +int log_init(); + +#define log_set_prefix(base_path, filename_prefix) \ + log_set_prefix_ex(&g_log_context, base_path, filename_prefix) + +#define log_set_filename(log_filename) \ + log_set_filename_ex(&g_log_context, log_filename) + +#define log_set_cache(bLogCache) log_set_cache_ex(&g_log_context, bLogCache) + +#define log_destroy() log_destroy_ex(&g_log_context) + +/** init function, use stderr for output by default + * parameters: + * pContext: the log context + * return: 0 for success, != 0 fail +*/ +int log_init_ex(LogContext *pContext); + +/** set log filename prefix, such as "tracker", the log filename will be + * ${base_path}/logs/tracker.log + * parameters: + * pContext: the log context + * base_path: base path + * filename_prefix: log filename prefix + * return: 0 for success, != 0 fail +*/ +int log_set_prefix_ex(LogContext *pContext, const char *base_path, \ + const char *filename_prefix); + +/** set log filename + * parameters: + * pContext: the log context + * log_filename: log filename + * return: 0 for success, != 0 fail +*/ +int log_set_filename_ex(LogContext *pContext, const char *log_filename); + +/** set if use log cache + * parameters: + * pContext: the log context + * bLogCache: true for cache in buffer, false directly write to disk + * return: none +*/ +void log_set_cache_ex(LogContext *pContext, const bool bLogCache); + +/** set time precision + * parameters: + * pContext: the log context + * time_precision: the time precision + * return: none +*/ +void log_set_time_precision(LogContext *pContext, const int time_precision); + +/** destroy function + * parameters: + * pContext: the log context + * bLogCache: true for cache in buffer, false directly write to disk + * return: none +*/ +void log_destroy_ex(LogContext *pContext); + +/** log to file + * parameters: + * pContext: the log context + * priority: unix priority + * format: printf format + * ...: arguments for printf format + * return: none +*/ +void log_it_ex(LogContext *pContext, const int priority, \ + const char *format, ...); + +/** log to file + * parameters: + * pContext: the log context + * priority: unix priority + * text: text string to log + * text_len: text string length (bytes) + * return: none +*/ +void log_it_ex1(LogContext *pContext, const int priority, \ + const char *text, const int text_len); + +/** sync log buffer to log file + * parameters: + * args: should be (LogContext *) + * return: error no, 0 for success, != 0 fail +*/ +int log_sync_func(void *args); + +/** set rotate flag to true + * parameters: + * args: should be (LogContext *) + * return: error no, 0 for success, != 0 fail +*/ +int log_notify_rotate(void *args); + +void logEmergEx(LogContext *pContext, const char *format, ...); +void logCritEx(LogContext *pContext, const char *format, ...); +void logAlertEx(LogContext *pContext, const char *format, ...); +void logErrorEx(LogContext *pContext, const char *format, ...); +void logWarningEx(LogContext *pContext, const char *format, ...); +void logNoticeEx(LogContext *pContext, const char *format, ...); +void logInfoEx(LogContext *pContext, const char *format, ...); +void logDebugEx(LogContext *pContext, const char *format, ...); +void logAccess(LogContext *pContext, struct timeval *tvStart, \ + const char *format, ...); + +//#define LOG_FORMAT_CHECK + +#ifdef LOG_FORMAT_CHECK /*only for format check*/ + +#define logEmerg printf +#define logCrit printf +#define logAlert printf +#define logError printf +#define logWarning printf +#define logNotice printf +#define logInfo printf +#define logDebug printf + +#else + +/* following functions use global log context: g_log_context */ +void logEmerg(const char *format, ...); +void logCrit(const char *format, ...); +void logAlert(const char *format, ...); +void logError(const char *format, ...); +void logWarning(const char *format, ...); +void logNotice(const char *format, ...); +void logInfo(const char *format, ...); +void logDebug(const char *format, ...); + +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/md5.c b/common/md5.c new file mode 100644 index 0000000..4ca8b77 --- /dev/null +++ b/common/md5.c @@ -0,0 +1,356 @@ +#include "md5.h" +#include +#include +#include + +#define S11 7 +#define S12 12 +#define S13 17 +#define S14 22 +#define S21 5 +#define S22 9 +#define S23 14 +#define S24 20 +#define S31 4 +#define S32 11 +#define S33 16 +#define S34 23 +#define S41 6 +#define S42 10 +#define S43 15 +#define S44 21 + +static void MD5Transform (UINT4[4], unsigned char[64]); +static void Encode (unsigned char *, UINT4 *, unsigned int); +static void Decode (UINT4 *, unsigned char *, unsigned int); +static void MD5_memcpy (POINTER, POINTER, unsigned int); +static void MD5_memset (POINTER, int, unsigned int); + +static unsigned char PADDING[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* + * F, G, H and I are basic MD5 functions. + */ +#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) +#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define I(x, y, z) ((y) ^ ((x) | (~z))) + +/* + * ROTATE_LEFT rotates x left n bits. + */ +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n)))) + +/* + * FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. Rotation is + * separate from addition to prevent recomputation. + */ +#define FF(a, b, c, d, x, s, ac) { \ +(a) += F ((b), (c), (d)) + (x) + (UINT4)(ac); \ +(a) = ROTATE_LEFT ((a), (s)); \ +(a) += (b); \ + } +#define GG(a, b, c, d, x, s, ac) { \ +(a) += G ((b), (c), (d)) + (x) + (UINT4)(ac); \ +(a) = ROTATE_LEFT ((a), (s)); \ +(a) += (b); \ + } +#define HH(a, b, c, d, x, s, ac) { \ +(a) += H ((b), (c), (d)) + (x) + (UINT4)(ac); \ +(a) = ROTATE_LEFT ((a), (s)); \ +(a) += (b); \ + } +#define II(a, b, c, d, x, s, ac) { \ +(a) += I ((b), (c), (d)) + (x) + (UINT4)(ac); \ +(a) = ROTATE_LEFT ((a), (s)); \ +(a) += (b); \ + } + +/* + * MD5 initialization. Begins an MD5 operation, writing a new context. + */ +void +my_md5_init(MD5_CTX *context) +{ + context->count[0] = context->count[1] = 0; + /* + * Load magic initialization constants. + */ + context->state[0] = 0x67452301; + context->state[1] = 0xefcdab89; + context->state[2] = 0x98badcfe; + context->state[3] = 0x10325476; +} + +/* + * MD5 block update operation. Continues an MD5 message-digest operation, + * processing another message block, and updating the context. + */ +void +my_md5_update(MD5_CTX *context, unsigned char *input, unsigned int inputLen) +{ + unsigned int i, index, partLen; + + /* Compute number of bytes mod 64 */ + index = (unsigned int) ((context->count[0] >> 3) & 0x3F); + + /* Update number of bits */ + if ((context->count[0] += ((UINT4) inputLen << 3)) < ((UINT4) inputLen << 3)) + context->count[1]++; + context->count[1] += ((UINT4) inputLen >> 29); + + partLen = 64 - index; + + /* + * Transform as many times as possible. + */ + if (inputLen >= partLen) { + MD5_memcpy + ((POINTER) & context->buffer[index], (POINTER) input, partLen); + MD5Transform(context->state, context->buffer); + + for (i = partLen; i + 63 < inputLen; i += 64) + MD5Transform(context->state, &input[i]); + + index = 0; + } else + i = 0; + + /* Buffer remaining input */ + MD5_memcpy + ((POINTER) & context->buffer[index], (POINTER) & input[i], + inputLen - i); +} + +/* + * MD5 finalization. Ends an MD5 message-digest operation, writing the the + * message digest and zeroizing the context. + */ +void +my_md5_final(unsigned char digest[16], MD5_CTX *context) +{ + unsigned char bits[8]; + unsigned int index, padLen; + + /* Save number of bits */ + Encode(bits, context->count, 8); + + /* + * Pad out to 56 mod 64. + */ + index = (unsigned int) ((context->count[0] >> 3) & 0x3f); + padLen = (index < 56) ? (56 - index) : (120 - index); + my_md5_update(context, PADDING, padLen); + + /* Append length (before padding) */ + my_md5_update(context, bits, 8); + /* Store state in digest */ + Encode(digest, context->state, 16); + + /* + * Zeroize sensitive information. + */ + MD5_memset((POINTER) context, 0, sizeof(*context)); +} + +/* + * MD5 basic transformation. Transforms state based on block. + */ +static void +MD5Transform(UINT4 state[4], unsigned char block[64]) +{ + UINT4 a = state[0], b = state[1], c = state[2], d = state[3], + x[16]; + + Decode(x, block, 64); + + /* Round 1 */ + FF(a, b, c, d, x[0], S11, 0xd76aa478); /* 1 */ + FF(d, a, b, c, x[1], S12, 0xe8c7b756); /* 2 */ + FF(c, d, a, b, x[2], S13, 0x242070db); /* 3 */ + FF(b, c, d, a, x[3], S14, 0xc1bdceee); /* 4 */ + FF(a, b, c, d, x[4], S11, 0xf57c0faf); /* 5 */ + FF(d, a, b, c, x[5], S12, 0x4787c62a); /* 6 */ + FF(c, d, a, b, x[6], S13, 0xa8304613); /* 7 */ + FF(b, c, d, a, x[7], S14, 0xfd469501); /* 8 */ + FF(a, b, c, d, x[8], S11, 0x698098d8); /* 9 */ + FF(d, a, b, c, x[9], S12, 0x8b44f7af); /* 10 */ + FF(c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ + FF(b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ + FF(a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ + FF(d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ + FF(c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ + FF(b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ + + /* Round 2 */ + GG(a, b, c, d, x[1], S21, 0xf61e2562); /* 17 */ + GG(d, a, b, c, x[6], S22, 0xc040b340); /* 18 */ + GG(c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ + GG(b, c, d, a, x[0], S24, 0xe9b6c7aa); /* 20 */ + GG(a, b, c, d, x[5], S21, 0xd62f105d); /* 21 */ + GG(d, a, b, c, x[10], S22, 0x2441453); /* 22 */ + GG(c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ + GG(b, c, d, a, x[4], S24, 0xe7d3fbc8); /* 24 */ + GG(a, b, c, d, x[9], S21, 0x21e1cde6); /* 25 */ + GG(d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ + GG(c, d, a, b, x[3], S23, 0xf4d50d87); /* 27 */ + GG(b, c, d, a, x[8], S24, 0x455a14ed); /* 28 */ + GG(a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ + GG(d, a, b, c, x[2], S22, 0xfcefa3f8); /* 30 */ + GG(c, d, a, b, x[7], S23, 0x676f02d9); /* 31 */ + GG(b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ + + /* Round 3 */ + HH(a, b, c, d, x[5], S31, 0xfffa3942); /* 33 */ + HH(d, a, b, c, x[8], S32, 0x8771f681); /* 34 */ + HH(c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ + HH(b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ + HH(a, b, c, d, x[1], S31, 0xa4beea44); /* 37 */ + HH(d, a, b, c, x[4], S32, 0x4bdecfa9); /* 38 */ + HH(c, d, a, b, x[7], S33, 0xf6bb4b60); /* 39 */ + HH(b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ + HH(a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ + HH(d, a, b, c, x[0], S32, 0xeaa127fa); /* 42 */ + HH(c, d, a, b, x[3], S33, 0xd4ef3085); /* 43 */ + HH(b, c, d, a, x[6], S34, 0x4881d05); /* 44 */ + HH(a, b, c, d, x[9], S31, 0xd9d4d039); /* 45 */ + HH(d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ + HH(c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ + HH(b, c, d, a, x[2], S34, 0xc4ac5665); /* 48 */ + + /* Round 4 */ + II(a, b, c, d, x[0], S41, 0xf4292244); /* 49 */ + II(d, a, b, c, x[7], S42, 0x432aff97); /* 50 */ + II(c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ + II(b, c, d, a, x[5], S44, 0xfc93a039); /* 52 */ + II(a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ + II(d, a, b, c, x[3], S42, 0x8f0ccc92); /* 54 */ + II(c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ + II(b, c, d, a, x[1], S44, 0x85845dd1); /* 56 */ + II(a, b, c, d, x[8], S41, 0x6fa87e4f); /* 57 */ + II(d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ + II(c, d, a, b, x[6], S43, 0xa3014314); /* 59 */ + II(b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ + II(a, b, c, d, x[4], S41, 0xf7537e82); /* 61 */ + II(d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ + II(c, d, a, b, x[2], S43, 0x2ad7d2bb); /* 63 */ + II(b, c, d, a, x[9], S44, 0xeb86d391); /* 64 */ + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + + /* + * Zeroize sensitive information. + * + */ + MD5_memset((POINTER) x, 0, sizeof(x)); +} + +/* + * Encodes input (UINT4) into output (unsigned char). Assumes len is a + * multiple of 4. + */ +static void +Encode(unsigned char *output, UINT4 *input, unsigned int len) +{ + unsigned int i, j; + + for (i = 0, j = 0; j < len; i++, j += 4) { + output[j] = (unsigned char) (input[i] & 0xff); + output[j + 1] = (unsigned char) ((input[i] >> 8) & 0xff); + output[j + 2] = (unsigned char) ((input[i] >> 16) & 0xff); + output[j + 3] = (unsigned char) ((input[i] >> 24) & 0xff); + } +} + +/* + * Decodes input (unsigned char) into output (UINT4). Assumes len is a + * multiple of 4. + */ +static void +Decode(UINT4 *output, unsigned char *input, unsigned int len) +{ + unsigned int i, j; + + for (i = 0, j = 0; j < len; i++, j += 4) + output[i] = ((UINT4) input[j]) | (((UINT4) input[j + 1]) << 8) | + (((UINT4) input[j + 2]) << 16) | (((UINT4) input[j + 3]) << 24); +} + +/* + * Note: Replace "for loop" with standard memcpy if possible. + */ + +static void +MD5_memcpy(POINTER output, POINTER input, unsigned int len) +{ + unsigned int i; + + for (i = 0; i < len; i++) + output[i] = input[i]; +} + +/* + * Note: Replace "for loop" with standard memset if possible. + */ +static void +MD5_memset(POINTER output, int value, unsigned int len) +{ + unsigned int i; + + for (i = 0; i < len; i++) + ((char *) output)[i] = (char) value; +} + +/* + * Digests a string + */ +int my_md5_string(char *string,unsigned char digest[16]) +{ + MD5_CTX context; + unsigned int len = strlen(string); + + my_md5_init(&context); + my_md5_update(&context, (unsigned char *)string, len); + my_md5_final(digest, &context); + return 0; +} + +int my_md5_buffer(char *buffer, unsigned int len, unsigned char digest[16]) +{ + MD5_CTX context; + + my_md5_init(&context); + my_md5_update(&context, (unsigned char *)buffer, len); + my_md5_final(digest, &context); + return 0; +} + +int my_md5_file(char *filename,unsigned char digest[16]) +{ + FILE *file; + MD5_CTX context; + int len; + unsigned char buffer[1024]; + + if ((file = fopen(filename, "rb")) == NULL) + return -1; + else { + my_md5_init(&context); + while ((len = fread(buffer, 1, 1024, file)) > 0) + { + my_md5_update(&context, buffer, len); + } + my_md5_final(digest, &context); + + fclose(file); + } + return 0; +} + diff --git a/common/md5.h b/common/md5.h new file mode 100644 index 0000000..8998286 --- /dev/null +++ b/common/md5.h @@ -0,0 +1,55 @@ +#ifndef MCL_MD5_H +#define MCL_MD5_H +#include + +typedef unsigned char *POINTER; +typedef unsigned short int UINT2; +typedef unsigned int UINT4; + +typedef struct { + UINT4 state[4]; /* state (ABCD) */ + UINT4 count[2]; /* number of bits, modulo 2^64 (lsb first) */ + unsigned char buffer[64]; /* input buffer */ +} MD5_CTX; + +#ifdef __cplusplus +extern "C" { +#endif + +/** md5 for string + * parameters: + * string: the string to md5 + * digest: store the md5 digest + * return: 0 for success, != 0 fail +*/ +int my_md5_string(char *string, unsigned char digest[16]); + +/** md5 for file + * parameters: + * filename: the filename whose content to md5 + * digest: store the md5 digest + * return: 0 for success, != 0 fail +*/ +int my_md5_file(char *filename, unsigned char digest[16]); + +/** md5 for buffer + * parameters: + * buffer: the buffer to md5 + * len: the buffer length + * digest: store the md5 digest + * return: 0 for success, != 0 fail +*/ +int my_md5_buffer(char *buffer, unsigned int len, unsigned char digest[16]); + +void my_md5_init (MD5_CTX *); + +void my_md5_update (MD5_CTX *, unsigned char *, unsigned int); + +void my_md5_final (unsigned char [16], MD5_CTX *); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/mime_file_parser.c b/common/mime_file_parser.c new file mode 100644 index 0000000..c28f336 --- /dev/null +++ b/common/mime_file_parser.c @@ -0,0 +1,131 @@ + +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "http_func.h" +#include "shared_func.h" +#include "mime_file_parser.h" + +int load_mime_types_from_file(HashArray *pHash, const char *mime_filename) +{ +#define MIME_DELIM_CHARS " \t" + + int result; + char *content; + char *pLine; + char *pLastEnd; + char *content_type; + char *ext_name; + char *lasts; + int http_status; + int content_len; + int64_t file_size; + char error_info[512]; + + if (strncasecmp(mime_filename, "http://", 7) == 0) + { + if ((result=get_url_content(mime_filename, 30, 60, &http_status,\ + &content, &content_len, error_info)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "get_url_content fail, " \ + "url: %s, error info: %s", \ + __LINE__, mime_filename, error_info); + return result; + } + + if (http_status != 200) + { + free(content); + logError("file: "__FILE__", line: %d, " \ + "HTTP status code: %d != 200, url: %s", \ + __LINE__, http_status, mime_filename); + return EINVAL; + } + } + else + { + if ((result=getFileContent(mime_filename, &content, \ + &file_size)) != 0) + { + return result; + } + } + + if ((result=hash_init_ex(pHash, PJWHash, 2 * 1024, 0.75, 0, true)) != 0) + { + free(content); + logError("file: "__FILE__", line: %d, " \ + "hash_init_ex fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + pLastEnd = content - 1; + while (pLastEnd != NULL) + { + pLine = pLastEnd + 1; + pLastEnd = strchr(pLine, '\n'); + if (pLastEnd != NULL) + { + *pLastEnd = '\0'; + } + + if (*pLine == '\0' || *pLine == '#') + { + continue; + } + + lasts = NULL; + content_type = strtok_r(pLine, MIME_DELIM_CHARS, &lasts); + while (1) + { + ext_name = strtok_r(NULL, MIME_DELIM_CHARS, &lasts); + if (ext_name == NULL) + { + break; + } + + if (*ext_name == '\0') + { + continue; + } + + if ((result=hash_insert_ex(pHash, ext_name, \ + strlen(ext_name)+1, content_type, \ + strlen(content_type)+1, true)) < 0) + { + free(content); + result *= -1; + logError("file: "__FILE__", line: %d, " \ + "hash_insert_ex fail, errno: %d, " \ + "error info: %s", __LINE__, \ + result, STRERROR(result)); + return result; + } + } + } + + free(content); + + //hash_stat_print(pHash); + return 0; +} + diff --git a/common/mime_file_parser.h b/common/mime_file_parser.h new file mode 100644 index 0000000..621ae3a --- /dev/null +++ b/common/mime_file_parser.h @@ -0,0 +1,42 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef _MINE_FILE_PARSER_H +#define _MINE_FILE_PARSER_H + +#include +#include +#include +#include +#include +#include "hash.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** +load mime types from file +params: + pHash: hash array to store the mime types, + key is the file extension name, eg. jpg + value is the content type, eg. image/jpeg + the hash array will be initialized in this function, + the hash array should be destroyed when used done + mime_filename: the mime filename, + file format is same as apache's file: mime.types +return: 0 for success, !=0 for fail +**/ +int load_mime_types_from_file(HashArray *pHash, const char *mime_filename); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/process_ctrl.c b/common/process_ctrl.c new file mode 100644 index 0000000..e52b06e --- /dev/null +++ b/common/process_ctrl.c @@ -0,0 +1,234 @@ +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "logger.h" +#include "process_ctrl.h" + +int get_pid_from_file(const char *pidFilename, pid_t *pid) +{ + char buff[32]; + int64_t file_size; + int result; + + if (access(pidFilename, F_OK) != 0) { + return errno != 0 ? errno : EPERM; + } + + file_size = sizeof(buff) - 1; + if ((result=getFileContentEx(pidFilename, buff, 0, &file_size)) != 0) { + return result; + } + + *(buff + file_size) = '\0'; + *pid = strtol(buff, NULL, 10); + if (*pid == 0) { + return EINVAL; + } + + return 0; +} + +int write_to_pid_file(const char *pidFilename) +{ + char buff[32]; + int len; + + len = sprintf(buff, "%d", (int)getpid()); + return writeToFile(pidFilename, buff, len); +} + +int delete_pid_file(const char *pidFilename) +{ + if (unlink(pidFilename) == 0) { + return 0; + } + else { + return errno != 0 ? errno : ENOENT; + } +} + +static int do_stop(const char *pidFilename, const bool bShowError, pid_t *pid) +{ + int result; + + if ((result=get_pid_from_file(pidFilename, pid)) != 0) { + if (bShowError) { + if (result == ENOENT) { + fprintf(stderr, "pid file: %s not exist!\n", pidFilename); + } + else { + fprintf(stderr, "get pid from file: %s fail, " \ + "errno: %d, error info: %s\n", + pidFilename, result, strerror(result)); + } + } + + return result; + } + + if (kill(*pid, SIGTERM) == 0) { + return 0; + } + else { + result = errno != 0 ? errno : EPERM; + if (bShowError || result != ESRCH) { + fprintf(stderr, "kill pid: %d fail, errno: %d, error info: %s\n", + (int)*pid, result, strerror(result)); + } + return result; + } +} + +int process_stop(const char *pidFilename) +{ + pid_t pid; + int result; + + result = do_stop(pidFilename, true, &pid); + if (result != 0) { + return result; + } + + fprintf(stderr, "waiting for pid [%d] exit ...\n", (int)pid); + do { + sleep(1); + } while (kill(pid, SIGTERM) == 0); + fprintf(stderr, "pid [%d] exit.\n", (int)pid); + + return 0; +} + +int process_restart(const char *pidFilename) +{ + int result; + pid_t pid; + + result = do_stop(pidFilename, false, &pid); + if (result == 0) { + fprintf(stderr, "waiting for pid [%d] exit ...\n", (int)pid); + do { + sleep(1); + } while (kill(pid, SIGTERM) == 0); + fprintf(stderr, "starting ...\n"); + } + + if (result == ENOENT || result == ESRCH) { + return 0; + } + + return result; +} + +int process_exist(const char *pidFilename) +{ + pid_t pid; + int result; + + if ((result=get_pid_from_file(pidFilename, &pid)) != 0) { + if (result == ENOENT) { + return false; + } + else { + fprintf(stderr, "get pid from file: %s fail, " \ + "errno: %d, error info: %s\n", + pidFilename, result, strerror(result)); + return true; + } + } + + if (kill(pid, 0) == 0) { + return true; + } + else if (errno == ENOENT || errno == ESRCH) { + return false; + } + else { + fprintf(stderr, "kill pid: %d fail, errno: %d, error info: %s\n", + (int)pid, errno, strerror(errno)); + return true; + } +} + +int get_base_path_from_conf_file(const char *filename, char *base_path, + const int path_size) +{ + char *pBasePath; + IniContext iniContext; + int result; + + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, ret code: %d", \ + __LINE__, filename, result); + return result; + } + + do + { + pBasePath = iniGetStrValue(NULL, "base_path", &iniContext); + if (pBasePath == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" must have item " \ + "\"base_path\"!", __LINE__, filename); + result = ENOENT; + break; + } + + snprintf(base_path, path_size, "%s", pBasePath); + chopPath(base_path); + if (!fileExists(base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" can't be accessed, error info: %s", \ + __LINE__, base_path, STRERROR(errno)); + result = errno != 0 ? errno : ENOENT; + break; + } + if (!isDir(base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" is not a directory!", \ + __LINE__, base_path); + result = ENOTDIR; + break; + } + } while (0); + + iniFreeContext(&iniContext); + return result; +} + +int process_action(const char *pidFilename, const char *action, bool *stop) +{ + *stop = false; + if (action == NULL) + { + return 0; + } + + if (strcmp(action, "stop") == 0) + { + *stop = true; + return process_stop(pidFilename); + } + else if (strcmp(action, "restart") == 0) + { + return process_restart(pidFilename); + } + else if (strcmp(action, "start") == 0) + { + return 0; + } + else + { + fprintf(stderr, "invalid action: %s\n", action); + return EINVAL; + } +} + diff --git a/common/process_ctrl.h b/common/process_ctrl.h new file mode 100644 index 0000000..e9f9fa1 --- /dev/null +++ b/common/process_ctrl.h @@ -0,0 +1,37 @@ + +#ifndef PROCESS_CTRL_H +#define PROCESS_CTRL_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int get_base_path_from_conf_file(const char *filename, char *base_path, + const int path_size); + +int get_pid_from_file(const char *pidFilename, pid_t *pid); + +int write_to_pid_file(const char *pidFilename); + +int delete_pid_file(const char *pidFilename); + +int process_stop(const char *pidFilename); + +int process_restart(const char *pidFilename); + +int process_exist(const char *pidFilename); + +int process_action(const char *pidFilename, const char *action, bool *stop); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/pthread_func.c b/common/pthread_func.c new file mode 100644 index 0000000..2ec04f4 --- /dev/null +++ b/common/pthread_func.c @@ -0,0 +1,193 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pthread_func.h" +#include "logger.h" + +int init_pthread_lock(pthread_mutex_t *pthread_lock) +{ + pthread_mutexattr_t mat; + int result; + + if ((result=pthread_mutexattr_init(&mat)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutexattr_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + if ((result=pthread_mutexattr_settype(&mat, \ + PTHREAD_MUTEX_ERRORCHECK)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutexattr_settype fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + if ((result=pthread_mutex_init(pthread_lock, &mat)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + if ((result=pthread_mutexattr_destroy(&mat)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call thread_mutexattr_destroy fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int init_pthread_attr(pthread_attr_t *pattr, const int stack_size) +{ + size_t old_stack_size; + size_t new_stack_size; + int result; + + if ((result=pthread_attr_init(pattr)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_attr_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if ((result=pthread_attr_getstacksize(pattr, &old_stack_size)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_attr_getstacksize fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if (stack_size > 0) + { + if (old_stack_size != stack_size) + { + new_stack_size = stack_size; + } + else + { + new_stack_size = 0; + } + } + else if (old_stack_size < 1 * 1024 * 1024) + { + new_stack_size = 1 * 1024 * 1024; + } + else + { + new_stack_size = 0; + } + + if (new_stack_size > 0) + { + if ((result=pthread_attr_setstacksize(pattr, \ + new_stack_size)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_attr_setstacksize fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + if ((result=pthread_attr_setdetachstate(pattr, \ + PTHREAD_CREATE_DETACHED)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_attr_setdetachstate fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int create_work_threads(int *count, void *(*start_func)(void *), \ + void *arg, pthread_t *tids, const int stack_size) +{ + int result; + pthread_attr_t thread_attr; + pthread_t *ptid; + pthread_t *ptid_end; + + if ((result=init_pthread_attr(&thread_attr, stack_size)) != 0) + { + return result; + } + + result = 0; + ptid_end = tids + (*count); + for (ptid=tids; ptid +#include +#include +#include +#include +#include +#include "common_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int init_pthread_lock(pthread_mutex_t *pthread_lock); +int init_pthread_attr(pthread_attr_t *pattr, const int stack_size); + +int create_work_threads(int *count, void *(*start_func)(void *), \ + void *arg, pthread_t *tids, const int stack_size); +int kill_work_threads(pthread_t *tids, const int count); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/sched_thread.c b/common/sched_thread.c new file mode 100644 index 0000000..58f67f5 --- /dev/null +++ b/common/sched_thread.c @@ -0,0 +1,508 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include "sched_thread.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "logger.h" + +volatile bool g_schedule_flag = false; +volatile time_t g_current_time = 0; + +static ScheduleArray waiting_schedule_array = {NULL, 0}; +static int waiting_del_id = -1; + +static int sched_cmp_by_next_call_time(const void *p1, const void *p2) +{ + return ((ScheduleEntry *)p1)->next_call_time - \ + ((ScheduleEntry *)p2)->next_call_time; +} + +static int sched_init_entries(ScheduleArray *pScheduleArray) +{ + ScheduleEntry *pEntry; + ScheduleEntry *pEnd; + time_t time_base; + struct tm tm_current; + struct tm tm_base; + + if (pScheduleArray->count < 0) + { + logError("file: "__FILE__", line: %d, " \ + "schedule count %d < 0", \ + __LINE__, pScheduleArray->count); + return EINVAL; + } + if (pScheduleArray->count == 0) + { + return 0; + } + + g_current_time = time(NULL); + localtime_r((time_t *)&g_current_time, &tm_current); + pEnd = pScheduleArray->entries + pScheduleArray->count; + for (pEntry=pScheduleArray->entries; pEntryinterval <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "shedule interval %d <= 0", \ + __LINE__, pEntry->interval); + return EINVAL; + } + + if (pEntry->time_base.hour == TIME_NONE) + { + pEntry->next_call_time = g_current_time + \ + pEntry->interval; + } + else + { + if (tm_current.tm_hour > pEntry->time_base.hour || \ + (tm_current.tm_hour == pEntry->time_base.hour \ + && tm_current.tm_min >= pEntry->time_base.minute)) + { + memcpy(&tm_base, &tm_current, sizeof(struct tm)); + } + else + { + time_base = g_current_time - 24 * 3600; + localtime_r(&time_base, &tm_base); + } + + tm_base.tm_hour = pEntry->time_base.hour; + tm_base.tm_min = pEntry->time_base.minute; + tm_base.tm_sec = 0; + time_base = mktime(&tm_base); + + pEntry->next_call_time = g_current_time + \ + pEntry->interval - (g_current_time - \ + time_base) % pEntry->interval; + } + + /* + { + char buff1[32]; + char buff2[32]; + logInfo("id=%d, current time=%s, first call time=%s\n", \ + pEntry->id, formatDatetime(g_current_time, \ + "%Y-%m-%d %H:%M:%S", buff1, sizeof(buff1)), \ + formatDatetime(pEntry->next_call_time, \ + "%Y-%m-%d %H:%M:%S", buff2, sizeof(buff2))); + } + */ + } + + return 0; +} + +static void sched_make_chain(ScheduleContext *pContext) +{ + ScheduleArray *pScheduleArray; + ScheduleEntry *pEntry; + + pScheduleArray = &(pContext->scheduleArray); + if (pScheduleArray->count == 0) + { + pContext->head = NULL; + pContext->tail = NULL; + return; + } + + qsort(pScheduleArray->entries, pScheduleArray->count, \ + sizeof(ScheduleEntry), sched_cmp_by_next_call_time); + + pContext->head = pScheduleArray->entries; + pContext->tail = pScheduleArray->entries + (pScheduleArray->count - 1); + for (pEntry=pScheduleArray->entries; pEntrytail; pEntry++) + { + pEntry->next = pEntry + 1; + } + pContext->tail->next = NULL; +} + +static int sched_check_waiting(ScheduleContext *pContext) +{ + ScheduleArray *pScheduleArray; + ScheduleEntry *newEntries; + ScheduleEntry *pWaitingEntry; + ScheduleEntry *pWaitingEnd; + ScheduleEntry *pSchedEntry; + ScheduleEntry *pSchedEnd; + int allocCount; + int newCount; + int result; + int deleteCount; + + pScheduleArray = &(pContext->scheduleArray); + deleteCount = 0; + if (waiting_del_id >= 0) + { + pSchedEnd = pScheduleArray->entries + pScheduleArray->count; + for (pSchedEntry=pScheduleArray->entries; \ + pSchedEntryid == waiting_del_id) + { + break; + } + } + + if (pSchedEntry < pSchedEnd) + { + pSchedEntry++; + while (pSchedEntry < pSchedEnd) + { + memcpy(pSchedEntry - 1, pSchedEntry, \ + sizeof(ScheduleEntry)); + pSchedEntry++; + } + + deleteCount++; + pScheduleArray->count--; + + logDebug("file: "__FILE__", line: %d, " \ + "delete task id: %d, " \ + "current schedule count: %d", __LINE__, \ + waiting_del_id, pScheduleArray->count); + } + + waiting_del_id = -1; + } + + if (waiting_schedule_array.count == 0) + { + if (deleteCount > 0) + { + sched_make_chain(pContext); + return 0; + } + + return ENOENT; + } + + allocCount = pScheduleArray->count + waiting_schedule_array.count; + newEntries = (ScheduleEntry *)malloc(sizeof(ScheduleEntry) * allocCount); + if (newEntries == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes failed, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(ScheduleEntry) * allocCount, \ + result, STRERROR(result)); + + if (deleteCount > 0) + { + sched_make_chain(pContext); + } + return result; + } + + if (pScheduleArray->count > 0) + { + memcpy(newEntries, pScheduleArray->entries, \ + sizeof(ScheduleEntry) * pScheduleArray->count); + } + newCount = pScheduleArray->count; + pWaitingEnd = waiting_schedule_array.entries + waiting_schedule_array.count; + for (pWaitingEntry=waiting_schedule_array.entries; \ + pWaitingEntryid == pSchedEntry->id) + { + memcpy(pSchedEntry, pWaitingEntry, \ + sizeof(ScheduleEntry)); + break; + } + } + + if (pSchedEntry == pSchedEnd) + { + memcpy(pSchedEntry, pWaitingEntry, \ + sizeof(ScheduleEntry)); + newCount++; + } + } + + logDebug("file: "__FILE__", line: %d, " \ + "schedule add entries: %d, replace entries: %d", + __LINE__, newCount - pScheduleArray->count, \ + waiting_schedule_array.count - (newCount - pScheduleArray->count)); + + if (pScheduleArray->entries != NULL) + { + free(pScheduleArray->entries); + } + pScheduleArray->entries = newEntries; + pScheduleArray->count = newCount; + + free(waiting_schedule_array.entries); + waiting_schedule_array.count = 0; + waiting_schedule_array.entries = NULL; + + sched_make_chain(pContext); + + return 0; +} + +static void *sched_thread_entrance(void *args) +{ + ScheduleContext *pContext; + ScheduleEntry *pPrevious; + ScheduleEntry *pCurrent; + ScheduleEntry *pSaveNext; + ScheduleEntry *pNode; + ScheduleEntry *pUntil; + int exec_count; + int i; + int sleep_time; + + pContext = (ScheduleContext *)args; + if (sched_init_entries(&(pContext->scheduleArray)) != 0) + { + free(pContext); + return NULL; + } + sched_make_chain(pContext); + + g_schedule_flag = true; + while (*(pContext->pcontinue_flag)) + { + sched_check_waiting(pContext); + if (pContext->scheduleArray.count == 0) //no schedule entry + { + sleep(1); + g_current_time = time(NULL); + continue; + } + + g_current_time = time(NULL); + sleep_time = pContext->head->next_call_time - g_current_time; + + /* + //fprintf(stderr, "count=%d, sleep_time=%d\n", \ + pContext->scheduleArray.count, sleep_time); + */ + while (sleep_time > 0 && *(pContext->pcontinue_flag)) + { + sleep(1); + g_current_time = time(NULL); + if (sched_check_waiting(pContext) == 0) + { + break; + } + sleep_time--; + } + + if (!(*(pContext->pcontinue_flag))) + { + break; + } + + exec_count = 0; + pCurrent = pContext->head; + while (*(pContext->pcontinue_flag) && (pCurrent != NULL \ + && pCurrent->next_call_time <= g_current_time)) + { + //fprintf(stderr, "exec task id=%d\n", pCurrent->id); + pCurrent->task_func(pCurrent->func_args); + pCurrent->next_call_time = g_current_time + \ + pCurrent->interval; + pCurrent = pCurrent->next; + exec_count++; + } + + if (exec_count == 0 || pContext->scheduleArray.count == 1) + { + continue; + } + + if (exec_count > pContext->scheduleArray.count / 2) + { + sched_make_chain(pContext); + continue; + } + + pNode = pContext->head; + pContext->head = pCurrent; //new chain head + for (i=0; inext_call_time >= pContext->tail->next_call_time) + { + pContext->tail->next = pNode; + pContext->tail = pNode; + pNode = pNode->next; + pContext->tail->next = NULL; + continue; + } + + pPrevious = NULL; + pUntil = pContext->head; + while (pUntil != NULL && \ + pNode->next_call_time > pUntil->next_call_time) + { + pPrevious = pUntil; + pUntil = pUntil->next; + } + + pSaveNext = pNode->next; + if (pPrevious == NULL) + { + pContext->head = pNode; + } + else + { + pPrevious->next = pNode; + } + pNode->next = pUntil; + + pNode = pSaveNext; + } + } + + g_schedule_flag = false; + + logDebug("file: "__FILE__", line: %d, " \ + "schedule thread exit", __LINE__); + + free(pContext); + return NULL; +} + +static int sched_dup_array(const ScheduleArray *pSrcArray, \ + ScheduleArray *pDestArray) +{ + int result; + int bytes; + + if (pSrcArray->count == 0) + { + pDestArray->entries = NULL; + pDestArray->count = 0; + return 0; + } + + bytes = sizeof(ScheduleEntry) * pSrcArray->count; + pDestArray->entries = (ScheduleEntry *)malloc(bytes); + if (pDestArray->entries == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes failed, " \ + "errno: %d, error info: %s", \ + __LINE__, bytes, result, STRERROR(result)); + return result; + } + + memcpy(pDestArray->entries, pSrcArray->entries, bytes); + pDestArray->count = pSrcArray->count; + return 0; +} + +int sched_add_entries(const ScheduleArray *pScheduleArray) +{ + int result; + + if (pScheduleArray->count == 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "no schedule entry", __LINE__); + return ENOENT; + } + + while (waiting_schedule_array.entries != NULL) + { + logDebug("file: "__FILE__", line: %d, " \ + "waiting for schedule array ready ...", __LINE__); + sleep(1); + } + + if ((result=sched_dup_array(pScheduleArray, &waiting_schedule_array))!=0) + { + return result; + } + + return sched_init_entries(&waiting_schedule_array); +} + +int sched_del_entry(const int id) +{ + if (id < 0) + { + logError("file: "__FILE__", line: %d, " \ + "id: %d is invalid!", __LINE__, id); + return EINVAL; + } + + while (waiting_del_id >= 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "waiting for delete ready ...", __LINE__); + sleep(1); + } + + waiting_del_id = id; + return 0; +} + +int sched_start(ScheduleArray *pScheduleArray, pthread_t *ptid, \ + const int stack_size, bool * volatile pcontinue_flag) +{ + int result; + pthread_attr_t thread_attr; + ScheduleContext *pContext; + + pContext = (ScheduleContext *)malloc(sizeof(ScheduleContext)); + if (pContext == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes failed, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(ScheduleContext), \ + result, STRERROR(result)); + return result; + } + + if ((result=init_pthread_attr(&thread_attr, stack_size)) != 0) + { + free(pContext); + return result; + } + + if ((result=sched_dup_array(pScheduleArray, \ + &(pContext->scheduleArray))) != 0) + { + free(pContext); + return result; + } + + pContext->pcontinue_flag = pcontinue_flag; + if ((result=pthread_create(ptid, &thread_attr, \ + sched_thread_entrance, pContext)) != 0) + { + free(pContext); + logError("file: "__FILE__", line: %d, " \ + "create thread failed, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + pthread_attr_destroy(&thread_attr); + return result; +} + diff --git a/common/sched_thread.h b/common/sched_thread.h new file mode 100644 index 0000000..9224a6f --- /dev/null +++ b/common/sched_thread.h @@ -0,0 +1,78 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef _SCHED_THREAD_H_ +#define _SCHED_THREAD_H_ + +#include +#include "common_define.h" + +typedef int (*TaskFunc) (void *args); + +typedef struct tagScheduleEntry +{ + int id; //the task id + + /* the time base to execute task, such as 00:00, interval is 3600, + means execute the task every hour as 1:00, 2:00, 3:00 etc. */ + TimeInfo time_base; + + int interval; //the interval for execute task, unit is second + + TaskFunc task_func; //callback function + void *func_args; //arguments pass to callback function + + /* following are internal fields, do not set manually! */ + time_t next_call_time; + struct tagScheduleEntry *next; +} ScheduleEntry; + +typedef struct +{ + ScheduleEntry *entries; + int count; +} ScheduleArray; + +typedef struct +{ + ScheduleArray scheduleArray; + ScheduleEntry *head; //schedule chain head + ScheduleEntry *tail; //schedule chain tail + bool *pcontinue_flag; +} ScheduleContext; + +#ifdef __cplusplus +extern "C" { +#endif + +extern volatile bool g_schedule_flag; //schedule continue running flag +extern volatile time_t g_current_time; //the current time + + +#define get_current_time() (g_schedule_flag ? g_current_time: time(NULL)) + +int sched_add_entries(const ScheduleArray *pScheduleArray); +int sched_del_entry(const int id); + +/** execute the schedule thread + * parameters: + * pScheduleArray: schedule task + * ptid: store the schedule thread id + * stack_size: set thread stack size (byes) + * pcontinue_flag: main process continue running flag + * return: error no, 0 for success, != 0 fail +*/ +int sched_start(ScheduleArray *pScheduleArray, pthread_t *ptid, \ + const int stack_size, bool * volatile pcontinue_flag); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/shared_func.c b/common/shared_func.c new file mode 100644 index 0000000..0096bf3 --- /dev/null +++ b/common/shared_func.c @@ -0,0 +1,2069 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "logger.h" +#include "sockopt.h" + +char *formatDatetime(const time_t nTime, \ + const char *szDateFormat, \ + char *buff, const int buff_size) +{ + static char szDateBuff[128]; + struct tm tmTime; + int size; + + localtime_r(&nTime, &tmTime); + if (buff == NULL) + { + buff = szDateBuff; + size = sizeof(szDateBuff); + } + else + { + size = buff_size; + } + + *buff = '\0'; + strftime(buff, size, szDateFormat, &tmTime); + + return buff; +} + +int getCharLen(const char *s) +{ + unsigned char *p; + int count = 0; + + p = (unsigned char *)s; + while (*p != '\0') + { + if (*p > 127) + { + if (*(++p) != '\0') + { + p++; + } + } + else + { + p++; + } + + count++; + } + + return count; +} + +char *replaceCRLF2Space(char *s) +{ + char *p = s; + + while (*p != '\0') + { + if (*p == '\r' || *p == '\n') + { + *p = ' '; + } + + p++; + } + + return s; +} + +char *getAbsolutePath(const char *filename, char *szAbsPath, \ + const int pathSize) +{ + char *p; + int nPathLen; + char szPath[1024]; + char cwd[256]; + + p = strrchr(filename, '/'); + if (p == NULL) + { + szPath[0] = '\0'; + } + else + { + nPathLen = p - filename; + if (nPathLen >= sizeof(szPath)) + { + logError("file: "__FILE__", line: %d, " \ + "filename length: %d is too long, exceeds %d",\ + __LINE__, nPathLen, (int)sizeof(szPath)); + return NULL; + } + + memcpy(szPath, filename, nPathLen); + szPath[nPathLen] = '\0'; + } + + if (szPath[0] == '/') + { + snprintf(szAbsPath, pathSize, "%s", szPath); + } + else + { + if (getcwd(cwd, sizeof(cwd)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "call getcwd fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return NULL; + } + + nPathLen = strlen(cwd); + if (cwd[nPathLen - 1] == '/') + { + cwd[nPathLen - 1] = '\0'; + } + + if (szPath[0] != '\0') + { + snprintf(szAbsPath, pathSize, "%s/%s", cwd, szPath); + } + else + { + snprintf(szAbsPath, pathSize, "%s", cwd); + } + } + + return szAbsPath; +} + +char *getExeAbsoluteFilename(const char *exeFilename, char *szAbsFilename, \ + const int maxSize) +{ + const char *filename; + const char *p; + int nFileLen; + int nPathLen; + char cwd[256]; + char szPath[1024]; + + nFileLen = strlen(exeFilename); + if (nFileLen >= sizeof(szPath)) + { + logError("file: "__FILE__", line: %d, " \ + "filename length: %d is too long, exceeds %d!", \ + __LINE__, nFileLen, (int)sizeof(szPath)); + return NULL; + } + + p = strrchr(exeFilename, '/'); + if (p == NULL) + { + int i; + char *search_paths[] = {"/bin", "/usr/bin", "/usr/local/bin"}; + + *szPath = '\0'; + filename = exeFilename; + for (i=0; i<3; i++) + { + snprintf(cwd, sizeof(cwd), "%s/%s", \ + search_paths[i], filename); + if (fileExists(cwd)) + { + strcpy(szPath, search_paths[i]); + break; + } + } + + if (*szPath == '\0') + { + if (!fileExists(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "can't find exe file %s!", __LINE__, \ + filename); + return NULL; + } + } + else + { + snprintf(szAbsFilename, maxSize, "%s/%s", \ + szPath, filename); + return szAbsFilename; + } + } + else + { + filename = p + 1; + nPathLen = p - exeFilename; + memcpy(szPath, exeFilename, nPathLen); + szPath[nPathLen] = '\0'; + } + + if (*szPath == '/') + { + snprintf(szAbsFilename, maxSize, "%s/%s", szPath, filename); + } + else + { + if (getcwd(cwd, sizeof(cwd)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "call getcwd fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return NULL; + } + + nPathLen = strlen(cwd); + if (cwd[nPathLen - 1] == '/') + { + cwd[nPathLen - 1] = '\0'; + } + + if (*szPath != '\0') + { + snprintf(szAbsFilename, maxSize, "%s/%s/%s", \ + cwd, szPath, filename); + } + else + { + snprintf(szAbsFilename, maxSize, "%s/%s", \ + cwd, filename); + } + } + + return szAbsFilename; +} + +#ifndef WIN32 +int getProccessCount(const char *progName, const bool bAllOwners) +{ + int *pids = NULL; + return getUserProcIds(progName, bAllOwners, pids, 0); +} + +int getUserProcIds(const char *progName, const bool bAllOwners, \ + int pids[], const int arrSize) +{ + char path[128]="/proc"; + char fullpath[128]; + struct stat statbuf; + struct dirent *dirp; + DIR *dp; + int myuid=getuid(); + int fd; + char filepath[128]; + char buf[256]; + char *ptr; + int nbytes; + char procname[64]; + int cnt=0; + char *pTargetProg; + + if ((dp = opendir(path)) == NULL) + { + return -1; + } + + pTargetProg = (char *)malloc(strlen(progName) + 1); + if (pTargetProg == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)strlen(progName) + 1); + return -1; + } + + ptr = strrchr(progName, '/'); + if (ptr == NULL) + { + strcpy(pTargetProg, progName); + } + else + { + strcpy(pTargetProg, ptr + 1); + } + + while( (dirp = readdir(dp)) != NULL ) + { + if (strcmp(dirp->d_name, ".")==0 || strcmp(dirp->d_name, "..")==0 ) + { + continue; + } + + sprintf(fullpath, "%s/%s", path, dirp->d_name); + memset(&statbuf, 0, sizeof(statbuf)); + if (lstat(fullpath, &statbuf)<0) + { + continue; + } + + if ((bAllOwners || (statbuf.st_uid == myuid)) && S_ISDIR(statbuf.st_mode)) + { + sprintf(filepath, "%s/cmdline", fullpath); + if ((fd = open(filepath, O_RDONLY))<0) + { + continue; + } + + memset(buf, 0, 256); + if ((nbytes = read(fd, buf, 255)) < 0){ + close(fd); + continue; + } + close(fd); + + if (*buf == '\0') + { + continue; + } + + ptr = strrchr(buf, '/'); + if (ptr == NULL) + { + snprintf(procname, 64, "%s", buf); + } + else + { + snprintf(procname, 64, "%s", ptr + 1); + } + + if (strcmp(procname, pTargetProg) == 0) + { + if (pids != NULL) + { + if (cnt >= arrSize) + { + break; + } + pids[cnt] = atoi(dirp->d_name); + } + + cnt++; + } + } + } + free(pTargetProg); + + closedir(dp); + return cnt; +} + +int getExecResult(const char *command, char *output, const int buff_size) +{ + FILE *fp; + char *pCurrent; + int bytes_read; + int remain_bytes; + + if((fp=popen(command, "r")) == NULL) + { + return errno != 0 ? errno : EMFILE; + } + + pCurrent = output; + remain_bytes = buff_size; + while (remain_bytes > 0 && \ + (bytes_read=fread(pCurrent, 1, remain_bytes, fp)) > 0) + { + pCurrent += bytes_read; + remain_bytes -= bytes_read; + } + + pclose(fp); + + if (remain_bytes <= 0) + { + return ENOSPC; + } + + *pCurrent = '\0'; + return 0; +} + +#endif + +char *toLowercase(char *src) +{ + char *p; + + p = src; + while (*p != '\0') + { + if (*p >= 'A' && *p <= 'Z') + { + *p += 32; + } + p++; + } + + return src; +} + +char *toUppercase(char *src) +{ + char *p; + + p = src; + while (*p != '\0') + { + if (*p >= 'a' && *p <= 'z') + { + *p -= 32; + } + p++; + } + + return src; +} + +void daemon_init(bool bCloseFiles) +{ +#ifndef WIN32 + pid_t pid; + int i; + + if((pid=fork()) != 0) + { + exit(0); + } + + setsid(); + + if((pid=fork()) != 0) + { + exit(0); + } + +#ifdef DEBUG_FLAG + #define MAX_CORE_FILE_SIZE (256 * 1024 * 1024) + if (set_rlimit(RLIMIT_CORE, MAX_CORE_FILE_SIZE) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "set max core dump file size to %d MB fail, " \ + "errno: %d, error info: %s", \ + __LINE__, MAX_CORE_FILE_SIZE / (1024 * 1024), \ + errno, STRERROR(errno)); + } +#else + if (chdir("/") != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "change directory to / fail, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } +#endif + + if (bCloseFiles) + { + for(i=0; i<=2; i++) + { + close(i); + } + } +#endif + + return; +} + +char *bin2hex(const char *s, const int len, char *szHexBuff) +{ + unsigned char *p; + unsigned char *pEnd; + int nLen; + + nLen = 0; + pEnd = (unsigned char *)s + len; + for (p=(unsigned char *)s; p=pStr; p--) + { + if (!(' ' == *p || '\n' == *p || '\r' == *p || '\t' == *p)) + { + break; + } + } + + if (p != pEnd) + { + *(p+1) = '\0'; + } + + return pStr; +} + +char *trim(char *pStr) +{ + trim_right(pStr); + trim_left(pStr); + return pStr; +} + +char *formatDateYYYYMMDDHHMISS(const time_t t, char *szDateBuff, const int nSize) +{ + time_t timer = t; + struct tm tm; + + localtime_r(&timer, &tm); + + snprintf(szDateBuff, nSize, "%04d%02d%02d%02d%02d%02d", \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec); + + return szDateBuff; +} + +int getOccurCount(const char *src, const char seperator) +{ + int count; + char *p; + + count = 0; + p = strchr(src, seperator); + while (p != NULL) + { + count++; + p = strchr(p + 1, seperator); + } + + return count; +} + +char **split(char *src, const char seperator, const int nMaxCols, int *nColCount) +{ + char **pCols; + char **pCurrent; + char *p; + int i; + int nLastIndex; + + if (src == NULL) + { + *nColCount = 0; + return NULL; + } + + *nColCount = 1; + p = strchr(src, seperator); + + while (p != NULL) + { + (*nColCount)++; + p = strchr(p + 1, seperator); + } + + if (nMaxCols > 0 && (*nColCount) > nMaxCols) + { + *nColCount = nMaxCols; + } + + pCurrent = pCols = (char **)malloc(sizeof(char *) * (*nColCount)); + if (pCols == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(char *) * (*nColCount)); + return NULL; + } + + p = src; + nLastIndex = *nColCount - 1; + for (i=0; i<*nColCount; i++) + { + *pCurrent = p; + pCurrent++; + + p = strchr(p, seperator); + if (i != nLastIndex) + { + *p = '\0'; + p++; + } + } + + return pCols; +} + +void freeSplit(char **p) +{ + if (p != NULL) + { + free(p); + } +} + +int splitEx(char *src, const char seperator, char **pCols, const int nMaxCols) +{ + char *p; + char **pCurrent; + int count = 0; + + if (nMaxCols <= 0) + { + return 0; + } + + p = src; + pCurrent = pCols; + + while (true) + { + *pCurrent = p; + pCurrent++; + + count++; + if (count >= nMaxCols) + { + break; + } + + p = strchr(p, seperator); + if (p == NULL) + { + break; + } + + *p = '\0'; + p++; + } + + return count; +} + +int my_strtok(char *src, const char *delim, char **pCols, const int nMaxCols) +{ + char *p; + char **pCurrent; + int count; + bool bWordEnd; + + if (src == NULL || pCols == NULL) + { + return -1; + } + + if (nMaxCols <= 0) + { + return 0; + } + + p = src; + pCurrent = pCols; + + while (*p != '\0') + { + if (strchr(delim, *p) == NULL) + { + break; + } + p++; + } + + if (*p == '\0') + { + return 0; + } + + *pCurrent = p; + bWordEnd = false; + count = 1; + if (count >= nMaxCols) + { + return count; + } + + while (*p != '\0') + { + if (strchr(delim, *p) != NULL) + { + *p = '\0'; + bWordEnd = true; + } + else + { + if (bWordEnd) + { + pCurrent++; + *pCurrent = p; + + count++; + if (count >= nMaxCols) + { + break; + } + + bWordEnd = false; + } + } + + p++; + } + + return count; +} + +int str_replace(const char *s, const int src_len, const char *replaced, + const char *new_str, char *dest, const int dest_size) +{ + const char *pStart; + const char *pEnd; + char *pDest; + const char *p; + int old_len; + int new_len; + int len; + int max_dest_len; + int remain_len; + + if (dest_size <= 0) + { + return 0; + } + + max_dest_len = dest_size - 1; + old_len = strlen(replaced); + new_len = strlen(new_str); + if (old_len == 0) + { + len = src_len < max_dest_len ? src_len : max_dest_len; + memcpy(dest, s, len); + dest[len] = '\0'; + return len; + } + + remain_len = max_dest_len; + pDest = dest; + pStart = s; + pEnd = s + src_len; + while (1) + { + p = strstr(pStart, replaced); + if (p == NULL) + { + break; + } + + len = p - pStart; + if (len > 0) + { + if (len < remain_len) + { + memcpy(pDest, pStart, len); + pDest += len; + remain_len -= len; + } + else + { + memcpy(pDest, pStart, remain_len); + pDest += remain_len; + *pDest = '\0'; + return pDest - dest; + } + } + + if (new_len < remain_len) + { + memcpy(pDest, new_str, new_len); + pDest += new_len; + remain_len -= new_len; + } + else + { + memcpy(pDest, new_str, remain_len); + pDest += remain_len; + *pDest = '\0'; + return pDest - dest; + } + + pStart = p + old_len; + } + + len = pEnd - pStart; + if (len > 0) + { + if (len > remain_len) + { + len = remain_len; + } + memcpy(pDest, pStart, len); + pDest += len; + } + *pDest = '\0'; + return pDest - dest; +} + +bool fileExists(const char *filename) +{ + return access(filename, 0) == 0; +} + +bool isDir(const char *filename) +{ + struct stat buf; + if (stat(filename, &buf) != 0) + { + return false; + } + + return S_ISDIR(buf.st_mode); +} + +bool isFile(const char *filename) +{ + struct stat buf; + if (stat(filename, &buf) != 0) + { + return false; + } + + return S_ISREG(buf.st_mode); +} + +void chopPath(char *filePath) +{ + int lastIndex; + if (*filePath == '\0') + { + return; + } + + lastIndex = strlen(filePath) - 1; + if (filePath[lastIndex] == '/') + { + filePath[lastIndex] = '\0'; + } +} + +int getFileContent(const char *filename, char **buff, int64_t *file_size) +{ + int fd; + + fd = open(filename, O_RDONLY); + if (fd < 0) + { + *buff = NULL; + *file_size = 0; + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if ((*file_size=lseek(fd, 0, SEEK_END)) < 0) + { + *buff = NULL; + *file_size = 0; + close(fd); + logError("file: "__FILE__", line: %d, " \ + "lseek file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + + *buff = (char *)malloc(*file_size + 1); + if (*buff == NULL) + { + *file_size = 0; + close(fd); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)(*file_size + 1)); + return errno != 0 ? errno : ENOMEM; + } + + if (lseek(fd, 0, SEEK_SET) < 0) + { + *buff = NULL; + *file_size = 0; + close(fd); + logError("file: "__FILE__", line: %d, " \ + "lseek file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + if (read(fd, *buff, *file_size) != *file_size) + { + free(*buff); + *buff = NULL; + *file_size = 0; + close(fd); + logError("file: "__FILE__", line: %d, " \ + "read from file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + + (*buff)[*file_size] = '\0'; + close(fd); + + return 0; +} + +int getFileContentEx(const char *filename, char *buff, \ + int64_t offset, int64_t *size) +{ + int fd; + int read_bytes; + + if (*size <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid size: "INT64_PRINTF_FORMAT, \ + __LINE__, *size); + return EINVAL; + } + + fd = open(filename, O_RDONLY); + if (fd < 0) + { + *size = 0; + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (offset > 0 && lseek(fd, offset, SEEK_SET) < 0) + { + *size = 0; + close(fd); + logError("file: "__FILE__", line: %d, " \ + "lseek file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + + if ((read_bytes=read(fd, buff, *size)) < 0) + { + *size = 0; + close(fd); + logError("file: "__FILE__", line: %d, " \ + "read from file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + + *size = read_bytes; + *(buff + (*size)) = '\0'; + close(fd); + + return 0; +} + +int writeToFile(const char *filename, const char *buff, const int file_size) +{ + int fd; + int result; + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + return result; + } + + if (write(fd, buff, file_size) != file_size) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + close(fd); + return result; + } + + if (fsync(fd) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "fsync file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + close(fd); + return result; + } + + close(fd); + return 0; +} + +int safeWriteToFile(const char *filename, const char *buff, \ + const int file_size) +{ + char tmpFilename[MAX_PATH_SIZE]; + int result; + + snprintf(tmpFilename, sizeof(tmpFilename), "%s.tmp", filename); + if ((result=writeToFile(tmpFilename, buff, file_size)) != 0) + { + return result; + } + + if (rename(tmpFilename, filename) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "rename file \"%s\" to \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, filename, \ + result, STRERROR(result)); + return result; + } + + return 0; +} + +void int2buff(const int n, char *buff) +{ + unsigned char *p; + p = (unsigned char *)buff; + *p++ = (n >> 24) & 0xFF; + *p++ = (n >> 16) & 0xFF; + *p++ = (n >> 8) & 0xFF; + *p++ = n & 0xFF; +} + +int buff2int(const char *buff) +{ + return (((unsigned char)(*buff)) << 24) | \ + (((unsigned char)(*(buff+1))) << 16) | \ + (((unsigned char)(*(buff+2))) << 8) | \ + ((unsigned char)(*(buff+3))); +} + +void long2buff(int64_t n, char *buff) +{ + unsigned char *p; + p = (unsigned char *)buff; + *p++ = (n >> 56) & 0xFF; + *p++ = (n >> 48) & 0xFF; + *p++ = (n >> 40) & 0xFF; + *p++ = (n >> 32) & 0xFF; + *p++ = (n >> 24) & 0xFF; + *p++ = (n >> 16) & 0xFF; + *p++ = (n >> 8) & 0xFF; + *p++ = n & 0xFF; +} + +int64_t buff2long(const char *buff) +{ + unsigned char *p; + p = (unsigned char *)buff; + return (((int64_t)(*p)) << 56) | \ + (((int64_t)(*(p+1))) << 48) | \ + (((int64_t)(*(p+2))) << 40) | \ + (((int64_t)(*(p+3))) << 32) | \ + (((int64_t)(*(p+4))) << 24) | \ + (((int64_t)(*(p+5))) << 16) | \ + (((int64_t)(*(p+6))) << 8) | \ + ((int64_t)(*(p+7))); +} + +int fd_gets(int fd, char *buff, const int size, int once_bytes) +{ + char *pDest; + char *p; + char *pEnd; + int read_bytes; + int remain_bytes; + int rewind_bytes; + + if (once_bytes <= 0) + { + once_bytes = 1; + } + + pDest = buff; + remain_bytes = size - 1; + while (remain_bytes > 0) + { + if (once_bytes > remain_bytes) + { + once_bytes = remain_bytes; + } + + read_bytes = read(fd, pDest, once_bytes); + if (read_bytes < 0) + { + return -1; + } + if (read_bytes == 0) + { + break; + } + + pEnd = pDest + read_bytes; + for (p=pDest; p= value)) + { + return 0; + } + + limit.rlim_cur = value; + if (setrlimit(resource, &limit) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call setrlimit fail, resource=%d, value=%d, " \ + "errno: %d, error info: %s", \ + __LINE__, resource, (int)value, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + return 0; +} + +bool is_filename_secure(const char *filename, const int len) +{ + if (len < 3) + { + return true; + } + + if (memcmp(filename, "../", 3) == 0) + { + return false; + } + + return (strstr(filename, "/../") == NULL); +} + +void load_log_level(IniContext *pIniContext) +{ + set_log_level(iniGetStrValue(NULL, "log_level", pIniContext)); +} + +int load_log_level_ex(const char *conf_filename) +{ + int result; + IniContext iniContext; + + if ((result=iniLoadFromFile(conf_filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, ret code: %d", \ + __LINE__, conf_filename, result); + return result; + } + + load_log_level(&iniContext); + iniFreeContext(&iniContext); + return 0; +} + +void set_log_level(char *pLogLevel) +{ + if (pLogLevel != NULL) + { + toUppercase(pLogLevel); + if ( strncmp(pLogLevel, "DEBUG", 5) == 0 || \ + strcmp(pLogLevel, "LOG_DEBUG") == 0) + { + g_log_context.log_level = LOG_DEBUG; + } + else if ( strncmp(pLogLevel, "INFO", 4) == 0 || \ + strcmp(pLogLevel, "LOG_INFO") == 0) + { + g_log_context.log_level = LOG_INFO; + } + else if ( strncmp(pLogLevel, "NOTICE", 6) == 0 || \ + strcmp(pLogLevel, "LOG_NOTICE") == 0) + { + g_log_context.log_level = LOG_NOTICE; + } + else if ( strncmp(pLogLevel, "WARN", 4) == 0 || \ + strcmp(pLogLevel, "LOG_WARNING") == 0) + { + g_log_context.log_level = LOG_WARNING; + } + else if ( strncmp(pLogLevel, "ERR", 3) == 0 || \ + strcmp(pLogLevel, "LOG_ERR") == 0) + { + g_log_context.log_level = LOG_ERR; + } + else if ( strncmp(pLogLevel, "CRIT", 4) == 0 || \ + strcmp(pLogLevel, "LOG_CRIT") == 0) + { + g_log_context.log_level = LOG_CRIT; + } + else if ( strncmp(pLogLevel, "ALERT", 5) == 0 || \ + strcmp(pLogLevel, "LOG_ALERT") == 0) + { + g_log_context.log_level = LOG_ALERT; + } + else if ( strncmp(pLogLevel, "EMERG", 5) == 0 || \ + strcmp(pLogLevel, "LOG_EMERG") == 0) + { + g_log_context.log_level = LOG_EMERG; + } + } +} + +int fd_add_flags(int fd, int adding_flags) +{ + int flags; + + flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) + { + logError("file: "__FILE__", line: %d, " \ + "fcntl fail, errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + if (fcntl(fd, F_SETFL, flags | adding_flags) == -1) + { + logError("file: "__FILE__", line: %d, " \ + "fcntl fail, errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +int set_run_by(const char *group_name, const char *username) +{ +#ifndef WIN32 + struct group *pGroup; + struct passwd *pUser; + int nErrNo; + if (group_name != NULL && *group_name != '\0') + { + pGroup = getgrnam(group_name); + if (pGroup == NULL) + { + nErrNo = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "getgrnam fail, errno: %d, error info: %s.", \ + __LINE__, nErrNo, STRERROR(nErrNo)); + return nErrNo; + } + + if (setegid(pGroup->gr_gid) != 0) + { + nErrNo = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "setegid fail, errno: %d, error info: %s.", \ + __LINE__, nErrNo, STRERROR(nErrNo)); + return nErrNo; + } + } + + if (username != NULL && *username != '\0') + { + pUser = getpwnam(username); + if (pUser == NULL) + { + nErrNo = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "getpwnam fail, errno: %d, error info: %s.", \ + __LINE__, nErrNo, STRERROR(nErrNo)); + return nErrNo; + } + + if (seteuid(pUser->pw_uid) != 0) + { + nErrNo = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "seteuid fail, errno: %d, error info: %s.", \ + __LINE__, nErrNo, STRERROR(nErrNo)); + return nErrNo; + } + } +#endif + + return 0; +} + +int load_allow_hosts(IniContext *pIniContext, \ + in_addr_t **allow_ip_addrs, int *allow_ip_count) +{ + int count; + IniItem *pItem; + IniItem *pItemStart; + IniItem *pItemEnd; + char *pItemValue; + char *pStart; + char *pEnd; + char *p; + char *pTail; + int alloc_count; + int nHeadLen; + int i; + in_addr_t addr; + char hostname[256]; + + if ((pItemStart=iniGetValuesEx(NULL, "allow_hosts", \ + pIniContext, &count)) == NULL) + { + *allow_ip_count = -1; /* -1 means match any ip address */ + *allow_ip_addrs = NULL; + return 0; + } + + pItemEnd = pItemStart + count; + for (pItem=pItemStart; pItemvalue, "*") == 0) + { + *allow_ip_count = -1; /* -1 means match any ip address*/ + *allow_ip_addrs = NULL; + return 0; + } + } + + alloc_count = count; + *allow_ip_count = 0; + *allow_ip_addrs = (in_addr_t *)malloc(sizeof(in_addr_t) * alloc_count); + if (*allow_ip_addrs == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s.", \ + __LINE__, (int)sizeof(in_addr_t) * alloc_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + for (pItem=pItemStart; pItemvalue) == '\0') + { + continue; + } + + pStart = strchr(pItem->value, '['); + if (pStart == NULL) + { + addr = getIpaddrByName(pItem->value, NULL, 0); + if (addr == INADDR_NONE) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid host name: %s", \ + __LINE__, pItem->value); + } + else + { + if (alloc_count < (*allow_ip_count) + 1) + { + alloc_count = (*allow_ip_count) + \ + (pItemEnd - pItem); + *allow_ip_addrs = (in_addr_t *)realloc( + *allow_ip_addrs, + sizeof(in_addr_t)*alloc_count); + if (*allow_ip_addrs == NULL) + { + logError("file: "__FILE__", line: %d, "\ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(in_addr_t) + * alloc_count, \ + errno, STRERROR(errno)); + + return errno != 0 ? errno : ENOMEM; + } + } + + (*allow_ip_addrs)[*allow_ip_count] = addr; + (*allow_ip_count)++; + } + + continue; + } + + + pEnd = strchr(pStart, ']'); + if (pEnd == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid host name: %s, expect \"]\"", \ + __LINE__, pItem->value); + continue; + } + + pItemValue = strdup(pItem->value); + if (pItemValue == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "strdup fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + continue; + } + + nHeadLen = pStart - pItem->value; + pStart = pItemValue + nHeadLen; + pEnd = pItemValue + (pEnd - pItem->value); + pTail = pEnd + 1; + + memcpy(hostname, pItem->value, nHeadLen); + p = pStart + 1; //skip [ + + while (p <= pEnd) + { + char *pNumStart1; + char *pNumStart2; + int nStart; + int nEnd; + int nNumLen1; + int nNumLen2; + char end_ch1; + char end_ch2; + char szFormat[16]; + + while (*p == ' ' || *p == '\t') //trim prior spaces + { + p++; + } + + pNumStart1 = p; + while (*p >='0' && *p <= '9') + { + p++; + } + + nNumLen1 = p - pNumStart1; + while (*p == ' ' || *p == '\t') //trim tail spaces + { + p++; + } + + if (!(*p == ',' || *p == '-' || *p == ']')) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid char \"%c\" in host name: %s",\ + __LINE__, *p, pItem->value); + break; + } + + end_ch1 = *p; + *(pNumStart1 + nNumLen1) = '\0'; + + if (nNumLen1 == 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid host name: %s, " \ + "empty entry before \"%c\"", \ + __LINE__, pItem->value, end_ch1); + break; + } + + nStart = atoi(pNumStart1); + if (end_ch1 == '-') + { + p++; //skip - + + /* trim prior spaces */ + while (*p == ' ' || *p == '\t') + { + p++; + } + + pNumStart2 = p; + while (*p >='0' && *p <= '9') + { + p++; + } + + nNumLen2 = p - pNumStart2; + /* trim tail spaces */ + while (*p == ' ' || *p == '\t') + { + p++; + } + + if (!(*p == ',' || *p == ']')) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid char \"%c\" in host name: %s",\ + __LINE__, *p, pItem->value); + break; + } + + end_ch2 = *p; + *(pNumStart2 + nNumLen2) = '\0'; + + if (nNumLen2 == 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid host name: %s, " \ + "empty entry before \"%c\"", \ + __LINE__, pItem->value, end_ch2); + break; + } + + nEnd = atoi(pNumStart2); + } + else + { + nEnd = nStart; + } + + if (alloc_count < *allow_ip_count+(nEnd - nStart + 1)) + { + alloc_count = *allow_ip_count + (nEnd - nStart) + + (pItemEnd - pItem); + *allow_ip_addrs = (in_addr_t *)realloc( + *allow_ip_addrs, + sizeof(in_addr_t)*alloc_count); + if (*allow_ip_addrs == NULL) + { + logError("file: "__FILE__", line: %d, "\ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, \ + (int)sizeof(in_addr_t) * \ + alloc_count,\ + errno, STRERROR(errno)); + + free(pItemValue); + return errno != 0 ? errno : ENOMEM; + } + } + + sprintf(szFormat, "%%0%dd%%s", nNumLen1); + for (i=nStart; i<=nEnd; i++) + { + sprintf(hostname + nHeadLen, szFormat, \ + i, pTail); + + addr = getIpaddrByName(hostname, NULL, 0); + if (addr == INADDR_NONE) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid host name: %s", \ + __LINE__, hostname); + } + else + { + (*allow_ip_addrs)[*allow_ip_count]=addr; + (*allow_ip_count)++; + } + + } + + p++; + } + + free(pItemValue); + } + + if (*allow_ip_count == 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "allow ip count: 0", __LINE__); + } + + if (*allow_ip_count > 0) + { + qsort(*allow_ip_addrs, *allow_ip_count, sizeof(in_addr_t), \ + cmp_by_ip_addr_t); + } + + /* + printf("*allow_ip_count=%d\n", *allow_ip_count); + for (i=0; i<*allow_ip_count; i++) + { + struct in_addr address; + address.s_addr = (*allow_ip_addrs)[i]; + printf("%s\n", inet_ntoa(address)); + } + */ + + return 0; +} + +int cmp_by_ip_addr_t(const void *p1, const void *p2) +{ + return memcmp((in_addr_t *)p1, (in_addr_t *)p2, sizeof(in_addr_t)); +} + +int parse_bytes(char *pStr, const int default_unit_bytes, int64_t *bytes) +{ + char *pReservedEnd; + + pReservedEnd = NULL; + *bytes = strtol(pStr, &pReservedEnd, 10); + if (*bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "bytes: "INT64_PRINTF_FORMAT" < 0", \ + __LINE__, *bytes); + return EINVAL; + } + + if (pReservedEnd == NULL || *pReservedEnd == '\0') + { + *bytes *= default_unit_bytes; + } + else if (*pReservedEnd == 'G' || *pReservedEnd == 'g') + { + *bytes *= 1024 * 1024 * 1024; + } + else if (*pReservedEnd == 'M' || *pReservedEnd == 'm') + { + *bytes *= 1024 * 1024; + } + else if (*pReservedEnd == 'K' || *pReservedEnd == 'k') + { + *bytes *= 1024; + } + + return 0; +} + +int set_rand_seed() +{ + struct timeval tv; + + if (gettimeofday(&tv, NULL) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call gettimeofday fail, " \ + "errno=%d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + srand(tv.tv_sec ^ tv.tv_usec); + return 0; +} + +int get_time_item_from_conf(IniContext *pIniContext, \ + const char *item_name, TimeInfo *pTimeInfo, \ + const byte default_hour, const byte default_minute) +{ + char *pValue; + int hour; + int minute; + + pValue = iniGetStrValue(NULL, item_name, pIniContext); + if (pValue == NULL) + { + pTimeInfo->hour = default_hour; + pTimeInfo->minute = default_minute; + return 0; + } + + if (sscanf(pValue, "%d:%d", &hour, &minute) != 2) + { + logError("file: "__FILE__", line: %d, " \ + "item \"%s\" 's value \"%s\" is not an valid time", \ + __LINE__, item_name, pValue); + return EINVAL; + } + + if ((hour < 0 || hour > 23) || (minute < 0 || minute > 59)) + { + logError("file: "__FILE__", line: %d, " \ + "item \"%s\" 's value \"%s\" is not an valid time", \ + __LINE__, item_name, pValue); + return EINVAL; + } + + pTimeInfo->hour = (byte)hour; + pTimeInfo->minute = (byte)minute; + + return 0; +} + +char *urlencode(const char *src, const int src_len, char *dest, int *dest_len) +{ + static unsigned char hex_chars[] = "0123456789ABCDEF"; + const unsigned char *pSrc; + const unsigned char *pEnd; + char *pDest; + + pDest = dest; + pEnd = (unsigned char *)src + src_len; + for (pSrc=(unsigned char *)src; pSrc= '0' && *pSrc <= '9') || + (*pSrc >= 'a' && *pSrc <= 'z') || + (*pSrc >= 'A' && *pSrc <= 'Z') || + (*pSrc == '_' || *pSrc == '-' || *pSrc == '.')) + { + *pDest++ = *pSrc; + } + else if (*pSrc == ' ') + { + *pDest++ = '+'; + } + else + { + *pDest++ = '%'; + *pDest++ = hex_chars[(*pSrc) >> 4]; + *pDest++ = hex_chars[(*pSrc) & 0x0F]; + } + } + + *pDest = '\0'; + *dest_len = pDest - dest; + + return dest; +} + +char *urldecode(const char *src, const int src_len, char *dest, int *dest_len) +{ +#define IS_HEX_CHAR(ch) \ + ((ch >= '0' && ch <= '9') || \ + (ch >= 'a' && ch <= 'f') || \ + (ch >= 'A' && ch <= 'F')) + +#define HEX_VALUE(ch, value) \ + if (ch >= '0' && ch <= '9') \ + { \ + value = ch - '0'; \ + } \ + else if (ch >= 'a' && ch <= 'f') \ + { \ + value = ch - 'a' + 10; \ + } \ + else \ + { \ + value = ch - 'A' + 10; \ + } + + const unsigned char *pSrc; + const unsigned char *pEnd; + char *pDest; + unsigned char cHigh; + unsigned char cLow; + int valHigh; + int valLow; + + pDest = dest; + pSrc = (unsigned char *)src; + pEnd = (unsigned char *)src + src_len; + while (pSrc < pEnd) + { + if (*pSrc == '%' && pSrc + 2 < pEnd) + { + cHigh = *(pSrc + 1); + cLow = *(pSrc + 2); + + if (IS_HEX_CHAR(cHigh) && IS_HEX_CHAR(cLow)) + { + HEX_VALUE(cHigh, valHigh) + HEX_VALUE(cLow, valLow) + *pDest++ = (valHigh << 4) | valLow; + pSrc += 3; + } + else + { + *pDest++ = *pSrc; + pSrc++; + } + } + else if (*pSrc == '+') + { + *pDest++ = ' '; + pSrc++; + } + else + { + *pDest++ = *pSrc; + pSrc++; + } + } + + *pDest = '\0'; + *dest_len = pDest - dest; + + return dest; +} + +int buffer_strcpy(BufferInfo *pBuff, const char *str) +{ + pBuff->length = strlen(str); + if (pBuff->alloc_size <= pBuff->length) + { + if (pBuff->buff != NULL) + { + free(pBuff->buff); + } + + pBuff->alloc_size = pBuff->length + 1; + pBuff->buff = (char *)malloc(pBuff->alloc_size); + if (pBuff->buff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pBuff->alloc_size, \ + errno, STRERROR(errno)); + pBuff->alloc_size = 0; + return errno != 0 ? errno : ENOMEM; + } + } + + memcpy(pBuff->buff, str, pBuff->length + 1); + return 0; +} + +int buffer_memcpy(BufferInfo *pBuff, const char *buff, const int len) +{ + pBuff->length = len; + if (pBuff->alloc_size <= pBuff->length) + { + if (pBuff->buff != NULL) + { + free(pBuff->buff); + } + + pBuff->alloc_size = pBuff->length; + pBuff->buff = (char *)malloc(pBuff->alloc_size); + if (pBuff->buff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pBuff->alloc_size, \ + errno, STRERROR(errno)); + pBuff->alloc_size = 0; + return errno != 0 ? errno : ENOMEM; + } + } + + memcpy(pBuff->buff, buff, pBuff->length); + return 0; +} + +int set_timer(const int first_remain_seconds, const int interval, \ + void (*sighandler)(int)) +{ + struct itimerval value; + struct sigaction act; + + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + act.sa_handler = sighandler; + if(sigaction(SIGALRM, &act, NULL) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + memset(&value, 0, sizeof(value)); + value.it_interval.tv_sec = interval; + value.it_value.tv_sec = first_remain_seconds; + if (setitimer(ITIMER_REAL, &value, NULL) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "call setitimer fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + return 0; +} + +int set_file_utimes(const char *filename, const time_t new_time) +{ + struct timeval tvs[2]; + + tvs[0].tv_sec = new_time; + tvs[0].tv_usec = 0; + tvs[1].tv_sec = new_time; + tvs[1].tv_usec = 0; + if (utimes(filename, tvs) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "call utimes file: %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, filename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + return 0; +} + +int ignore_signal_pipe() +{ + struct sigaction act; + + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + act.sa_handler = SIG_IGN; + if(sigaction(SIGPIPE, &act, NULL) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno; + } + + return 0; +} + diff --git a/common/shared_func.h b/common/shared_func.h new file mode 100644 index 0000000..971c0ad --- /dev/null +++ b/common/shared_func.h @@ -0,0 +1,495 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#ifndef SHARED_FUNC_H +#define SHARED_FUNC_H + +#include +#include +#include +#include +#include +#include "common_define.h" +#include "ini_file_reader.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** lowercase the string + * parameters: + * src: input string, will be changed + * return: lowercased string +*/ +char *toLowercase(char *src); + +/** uppercase the string + * parameters: + * src: input string, will be changed + * return: uppercased string +*/ +char *toUppercase(char *src); + + +/** date format to string + * parameters: + * nTime: unix timestamp + * szDateFormat: date format, more detail man strftime + * buff: store the formated result, can be NULL + * buff_size: buffer size, max bytes can contain + * return: formated date string +*/ +char *formatDatetime(const time_t nTime, \ + const char *szDateFormat, \ + char *buff, const int buff_size); + +/** get character count, only support GB charset + * parameters: + * s: the string + * return: character count +*/ +int getCharLen(const char *s); + +/** replace \r and \n to space + * parameters: + * s: the string + * return: replaced string +*/ +char *replaceCRLF2Space(char *s); + +/** get the filename absolute path + * parameters: + * fileame: the filename + * szAbsPath: store the absolute path + * pathSize: max bytes to contain + * return: absolute path, NULL for fail +*/ +char *getAbsolutePath(const char *fileame, char *szAbsPath, \ + const int pathSize); + +/** get the executable file absolute filename + * parameters: + * exeFilename: the executable filename + * szAbsFilename: store the absolute filename + * maxSize: max bytes to contain + * return: absolute filename, NULL for fail +*/ +char *getExeAbsoluteFilename(const char *exeFilename, char *szAbsFilename, \ + const int maxSize); + +#ifndef WIN32 + +/** get running process count by program name such as fdfs_trackerd + * parameters: + * progName: the program name + * bAllOwners: false for only get my proccess count + * return: proccess count, >= 0 success, < 0 fail +*/ +int getProccessCount(const char *progName, const bool bAllOwners); + +/** get running process ids by program name such as fdfs_trackerd + * parameters: + * progName: the program name + * bAllOwners: false for only get my proccess count + * pids: store the pids + * arrSize: max pids + * return: proccess count, >= 0 success, < 0 fail +*/ +int getUserProcIds(const char *progName, const bool bAllOwners, \ + int pids[], const int arrSize); + +/** execute program, get it's output + * parameters: + * command: the program + * output: store ouput result + * buff_size: output max size (bytes) + * return: error no, 0 success, != 0 fail +*/ +int getExecResult(const char *command, char *output, const int buff_size); + +#endif + +/** daemon init + * parameters: + * bCloseFiles: if close the stdin, stdout and stderr + * return: none +*/ +void daemon_init(bool bCloseFiles); + +/** convert buffer content to hex string such as 0B82A1 + * parameters: + * s: the buffer + * len: the buffer length + * szHexBuff: store the hex string (must have enough space) + * return: hex string (szHexBuff) +*/ +char *bin2hex(const char *s, const int len, char *szHexBuff); + +/** parse hex string to binary content + * parameters: + * s: the hex string such as 8B04CD + * szBinBuff: store the converted binary content(must have enough space) + * nDestLen: store the converted content length + * return: converted binary content (szBinBuff) +*/ +char *hex2bin(const char *s, char *szBinBuff, int *nDestLen); + +/** print binary buffer as hex string + * parameters: + * s: the buffer + * len: the buffer length + * return: none +*/ +void printBuffHex(const char *s, const int len); + +/** 32 bits int convert to buffer (big-endian) + * parameters: + * n: 32 bits int value + * buff: the buffer, at least 4 bytes space, no tail \0 + * return: none +*/ +void int2buff(const int n, char *buff); + +/** buffer convert to 32 bits int + * parameters: + * buff: big-endian 4 bytes buffer + * return: 32 bits int value +*/ +int buff2int(const char *buff); + +/** long (64 bits) convert to buffer (big-endian) + * parameters: + * n: 64 bits int value + * buff: the buffer, at least 8 bytes space, no tail \0 + * return: none +*/ +void long2buff(int64_t n, char *buff); + +/** buffer convert to 64 bits int + * parameters: + * buff: big-endian 8 bytes buffer + * return: 64 bits int value +*/ +int64_t buff2long(const char *buff); + +/** trim leading spaces ( \t\r\n) + * parameters: + * pStr: the string to trim + * return: trimed string porinter as pStr +*/ +char *trim_left(char *pStr); + +/** trim tail spaces ( \t\r\n) + * parameters: + * pStr: the string to trim + * return: trimed string porinter as pStr +*/ +char *trim_right(char *pStr); + +/** trim leading and tail spaces ( \t\r\n) + * parameters: + * pStr: the string to trim + * return: trimed string porinter as pStr +*/ +char *trim(char *pStr); + +/** copy string to BufferInfo + * parameters: + * pBuff: the dest buffer + * str: source string + * return: error no, 0 success, != 0 fail +*/ +int buffer_strcpy(BufferInfo *pBuff, const char *str); + +/** copy binary buffer to BufferInfo + * parameters: + * pBuff: the dest buffer + * buff: source buffer + * len: source buffer length + * return: error no, 0 success, != 0 fail +*/ +int buffer_memcpy(BufferInfo *pBuff, const char *buff, const int len); + +/** url encode + * parameters: + * src: the source string to encode + * src_len: source string length + * dest: store dest string + * dest_len: store the dest string length + * return: error no, 0 success, != 0 fail +*/ +char *urlencode(const char *src, const int src_len, char *dest, int *dest_len); + +/** url decode + * parameters: + * src: the source string to decode + * src_len: source string length + * dest: store dest string + * dest_len: store the dest string length + * return: error no, 0 success, != 0 fail +*/ +char *urldecode(const char *src, const int src_len, char *dest, int *dest_len); + +/** get char occurs count + * parameters: + * src: the source string + * seperator: find this char occurs times + * return: char occurs count +*/ +int getOccurCount(const char *src, const char seperator); + +/** split string + * parameters: + * src: the source string, will be modified by this function + * seperator: seperator char + * nMaxCols: max columns (max split count) + * nColCount: store the columns (array elements) count + * return: string array, should call freeSplit to free, return NULL when fail +*/ +char **split(char *src, const char seperator, const int nMaxCols, \ + int *nColCount); + +/** free split results + * parameters: + * p: return by function split + * return: none +*/ +void freeSplit(char **p); + + +/** split string + * parameters: + * src: the source string, will be modified by this function + * seperator: seperator char + * pCols: store split strings + * nMaxCols: max columns (max split count) + * return: string array / column count +*/ +int splitEx(char *src, const char seperator, char **pCols, const int nMaxCols); + +/** split string + * parameters: + * src: the source string, will be modified by this function + * seperator: seperator char + * pCols: store split strings + * nMaxCols: max columns (max split count) + * return: string array / column count +*/ +int my_strtok(char *src, const char *delim, char **pCols, const int nMaxCols); + +/** check file exist + * parameters: + * filename: the filename + * return: true if file exists, otherwise false +*/ +bool fileExists(const char *filename); + +/** check if a directory + * parameters: + * filename: the filename + * return: true for directory +*/ +bool isDir(const char *filename); + +/** check if a regular file + * parameters: + * filename: the filename + * return: true for regular file +*/ +bool isFile(const char *filename); + +/** check if filename securty, /../ ocur in filename not allowed + * parameters: + * filename: the filename + * len: filename length + * return: true for regular file +*/ +bool is_filename_secure(const char *filename, const int len); + +/** load log_level from config context + * parameters: + * pIniContext: the config context + * return: none +*/ +void load_log_level(IniContext *pIniContext); + +/** load log_level from config file + * parameters: + * conf_filename: the config filename + * return: none +*/ +int load_log_level_ex(const char *conf_filename); + +/** set global log level + * parameters: + * pLogLevel: log level string value + * return: none +*/ +void set_log_level(char *pLogLevel); + +/** load allow hosts from config context + * parameters: + * pIniContext: the config context + * allow_ip_addrs: store allow ip addresses + * allow_ip_count: store allow ip address count + * return: error no , 0 success, != 0 fail +*/ +int load_allow_hosts(IniContext *pIniContext, \ + in_addr_t **allow_ip_addrs, int *allow_ip_count); + +/** get time item from config context + * parameters: + * pIniContext: the config context + * item_name: item name in config file, time format: hour:minute, such as 15:25 + * pTimeInfo: store time info + * default_hour: default hour value + * default_minute: default minute value + * return: error no , 0 success, != 0 fail +*/ +int get_time_item_from_conf(IniContext *pIniContext, \ + const char *item_name, TimeInfo *pTimeInfo, \ + const byte default_hour, const byte default_minute); + +/** trim path tail char / + * parameters: + * filePath: the file path to chop + * return: none +*/ +void chopPath(char *filePath); + +/** get file content + * parameters: + * filename: the filename + * buff: return the buff, must be freed + * file_size: store the file size + * return: error no , 0 success, != 0 fail +*/ +int getFileContent(const char *filename, char **buff, int64_t *file_size); + +/** get file content + * parameters: + * filename: the filename + * buff: the buff to store file content + * offset: the start offset + * size: specify the size to fetch and return the fetched size + * return: error no , 0 success, != 0 fail +*/ +int getFileContentEx(const char *filename, char *buff, \ + int64_t offset, int64_t *size); + +/** write to file + * parameters: + * filename: the filename to write + * buff: the buffer to write + * file_size: the file size + * return: error no , 0 success, != 0 fail +*/ +int writeToFile(const char *filename, const char *buff, const int file_size); + +/** safe write to file, first write to tmp file, then rename to true filename + * parameters: + * filename: the filename to write + * buff: the buffer to write + * file_size: the file size + * return: error no , 0 success, != 0 fail +*/ +int safeWriteToFile(const char *filename, const char *buff, \ + const int file_size); + +/** get a line from file + * parameters: + * fd: the fd to read + * buff: the buffer to store the line + * size: the buffer max size + * once_bytes: the bytes per read + * return: error no , 0 success, != 0 fail +*/ +int fd_gets(int fd, char *buff, const int size, int once_bytes); + +/** set unix rlimit + * parameters: + * resource: resource id, please see sys/resource.h + * value: the value to set + * return: error no , 0 success, != 0 fail +*/ +int set_rlimit(int resource, const rlim_t value); + +/** set non block mode + * parameters: + * fd: the fd to set + * adding_flags: the flags to add + * return: error no , 0 success, != 0 fail +*/ +int fd_add_flags(int fd, int adding_flags); + +/** set non block mode + * parameters: + * fd: the fd to set + * return: error no , 0 success, != 0 fail +*/ +#define set_nonblock(fd) fd_add_flags(fd, O_NONBLOCK) + +/** set run by group and user + * parameters: + * group_name: the group name, can be NULL or empty + * username: the username, can be NULL or empty + * return: error no , 0 success, != 0 fail +*/ +int set_run_by(const char *group_name, const char *username); + +/** compare ip address, type is (in_addr_t *) + * parameters: + * p1: the first ip address + * p2: the second ip address + * return: > 0 when p1 > p2, 0 when p1 == p2, < 0 when p1 < p2 +*/ +int cmp_by_ip_addr_t(const void *p1, const void *p2); + +/** parse bytes + * parameters: + * pStr: the string to parse + * default_unit_bytes: default unit if not specified the unit like MB etc. + * bytes: store the parsed bytes + * return: error no , 0 success, != 0 fail +*/ +int parse_bytes(char *pStr, const int default_unit_bytes, int64_t *bytes); + +/** set rand seed + * return: error no , 0 success, != 0 fail +*/ +int set_rand_seed(); + +/** set timer wrapper + * parameters: + * first_remain_seconds: remain time for first time, in seconds + * interval: the interval + * sighandler: handler function + * return: error no , 0 success, != 0 fail +*/ +int set_timer(const int first_remain_seconds, const int interval, \ + void (*sighandler)(int)); + +/** set file access and modified times + * parameters: + * filename: the file to modify times + * new_time: the time to set + * return: error no , 0 success, != 0 fail +*/ +int set_file_utimes(const char *filename, const time_t new_time); + +/** ignore singal pipe (SIGPIPE) + * return: error no , 0 success, != 0 fail +*/ +int ignore_signal_pipe(); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/common/sockopt.c b/common/sockopt.c new file mode 100644 index 0000000..2a8dc42 --- /dev/null +++ b/common/sockopt.c @@ -0,0 +1,1744 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//socketopt.c +#include "common_define.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(OS_LINUX) || defined(OS_FREEBSD) +#include +#endif + +#include +#include +#include "shared_func.h" + +#ifdef OS_SUNOS +#include +#endif + +#ifdef USE_SENDFILE + +#ifdef OS_LINUX +#include +#else +#ifdef OS_FREEBSD +#include +#endif +#endif + +#endif + +#include "logger.h" +#include "hash.h" +#include "sockopt.h" + +#ifdef WIN32 +#define USE_SELECT +#else +#define USE_POLL +#endif + +#ifdef OS_LINUX +#ifndef TCP_KEEPIDLE +#define TCP_KEEPIDLE 4 /* Start keeplives after this period */ +#endif + +#ifndef TCP_KEEPINTVL +#define TCP_KEEPINTVL 5 /* Interval between keepalives */ +#endif + +#ifndef TCP_KEEPCNT +#define TCP_KEEPCNT 6 /* Number of keepalives before death */ +#endif +#endif + +int tcpgets(int sock, char* s, const int size, const int timeout) +{ + int result; + char t; + int i=1; + + if (s == NULL || size <= 0) + { + return EINVAL; + } + + while (i < size) + { + result = tcprecvdata(sock, &t, 1, timeout); + if (result != 0) + { + *s = 0; + return result; + } + + if (t == '\r') + { + continue; + } + + if (t == '\n') + { + *s = t; + s++; + *s = 0; + return 0; + } + + *s = t; + s++,i++; + } + + *s = 0; + return 0; +} + +int tcprecvdata_ex(int sock, void *data, const int size, \ + const int timeout, int *count) +{ + int left_bytes; + int read_bytes; + int res; + int ret_code; + unsigned char* p; +#ifdef USE_SELECT + fd_set read_set; + struct timeval t; +#else + struct pollfd pollfds; +#endif + +#ifdef USE_SELECT + FD_ZERO(&read_set); + FD_SET(sock, &read_set); +#else + pollfds.fd = sock; + pollfds.events = POLLIN; +#endif + + read_bytes = 0; + ret_code = 0; + p = (unsigned char*)data; + left_bytes = size; + while (left_bytes > 0) + { + +#ifdef USE_SELECT + if (timeout <= 0) + { + res = select(sock+1, &read_set, NULL, NULL, NULL); + } + else + { + t.tv_usec = 0; + t.tv_sec = timeout; + res = select(sock+1, &read_set, NULL, NULL, &t); + } +#else + res = poll(&pollfds, 1, 1000 * timeout); + if (pollfds.revents & POLLHUP) + { + ret_code = ENOTCONN; + break; + } +#endif + + if (res < 0) + { + ret_code = errno != 0 ? errno : EINTR; + break; + } + else if (res == 0) + { + ret_code = ETIMEDOUT; + break; + } + + read_bytes = recv(sock, p, left_bytes, 0); + if (read_bytes < 0) + { + ret_code = errno != 0 ? errno : EINTR; + break; + } + if (read_bytes == 0) + { + ret_code = ENOTCONN; + break; + } + + left_bytes -= read_bytes; + p += read_bytes; + } + + if (count != NULL) + { + *count = size - left_bytes; + } + + return ret_code; +} + +int tcpsenddata(int sock, void* data, const int size, const int timeout) +{ + int left_bytes; + int write_bytes; + int result; + unsigned char* p; +#ifdef USE_SELECT + fd_set write_set; + struct timeval t; +#else + struct pollfd pollfds; +#endif + +#ifdef USE_SELECT + FD_ZERO(&write_set); + FD_SET(sock, &write_set); +#else + pollfds.fd = sock; + pollfds.events = POLLOUT; +#endif + + p = (unsigned char*)data; + left_bytes = size; + while (left_bytes > 0) + { +#ifdef USE_SELECT + if (timeout <= 0) + { + result = select(sock+1, NULL, &write_set, NULL, NULL); + } + else + { + t.tv_usec = 0; + t.tv_sec = timeout; + result = select(sock+1, NULL, &write_set, NULL, &t); + } +#else + result = poll(&pollfds, 1, 1000 * timeout); + if (pollfds.revents & POLLHUP) + { + return ENOTCONN; + break; + } +#endif + + if (result < 0) + { + return errno != 0 ? errno : EINTR; + } + else if (result == 0) + { + return ETIMEDOUT; + } + + write_bytes = send(sock, p, left_bytes, 0); + if (write_bytes < 0) + { + return errno != 0 ? errno : EINTR; + } + + left_bytes -= write_bytes; + p += write_bytes; + } + + return 0; +} + +int tcprecvdata_nb_ex(int sock, void *data, const int size, \ + const int timeout, int *count) +{ + int left_bytes; + int read_bytes; + int res; + int ret_code; + unsigned char* p; +#ifdef USE_SELECT + fd_set read_set; + struct timeval t; +#else + struct pollfd pollfds; +#endif + +#ifdef USE_SELECT + FD_ZERO(&read_set); + FD_SET(sock, &read_set); +#else + pollfds.fd = sock; + pollfds.events = POLLIN; +#endif + + read_bytes = 0; + ret_code = 0; + p = (unsigned char*)data; + left_bytes = size; + while (left_bytes > 0) + { + read_bytes = recv(sock, p, left_bytes, 0); + if (read_bytes > 0) + { + left_bytes -= read_bytes; + p += read_bytes; + continue; + } + + if (read_bytes < 0) + { + + if (!(errno == EAGAIN || errno == EWOULDBLOCK)) + { + ret_code = errno != 0 ? errno : EINTR; + break; + } + } + else + { + ret_code = ENOTCONN; + break; + } + +#ifdef USE_SELECT + if (timeout <= 0) + { + res = select(sock+1, &read_set, NULL, NULL, NULL); + } + else + { + t.tv_usec = 0; + t.tv_sec = timeout; + res = select(sock+1, &read_set, NULL, NULL, &t); + } +#else + res = poll(&pollfds, 1, 1000 * timeout); + if (pollfds.revents & POLLHUP) + { + ret_code = ENOTCONN; + break; + } +#endif + + if (res < 0) + { + ret_code = errno != 0 ? errno : EINTR; + break; + } + else if (res == 0) + { + ret_code = ETIMEDOUT; + break; + } + } + + if (count != NULL) + { + *count = size - left_bytes; + } + + return ret_code; +} + +int tcpsenddata_nb(int sock, void* data, const int size, const int timeout) +{ + int left_bytes; + int write_bytes; + int result; + unsigned char* p; +#ifdef USE_SELECT + fd_set write_set; + struct timeval t; +#else + struct pollfd pollfds; +#endif + +#ifdef USE_SELECT + FD_ZERO(&write_set); + FD_SET(sock, &write_set); +#else + pollfds.fd = sock; + pollfds.events = POLLOUT; +#endif + + p = (unsigned char*)data; + left_bytes = size; + while (left_bytes > 0) + { + write_bytes = send(sock, p, left_bytes, 0); + if (write_bytes < 0) + { + if (!(errno == EAGAIN || errno == EWOULDBLOCK)) + { + return errno != 0 ? errno : EINTR; + } + } + else + { + left_bytes -= write_bytes; + p += write_bytes; + continue; + } + +#ifdef USE_SELECT + if (timeout <= 0) + { + result = select(sock+1, NULL, &write_set, NULL, NULL); + } + else + { + t.tv_usec = 0; + t.tv_sec = timeout; + result = select(sock+1, NULL, &write_set, NULL, &t); + } +#else + result = poll(&pollfds, 1, 1000 * timeout); + if (pollfds.revents & POLLHUP) + { + return ENOTCONN; + } +#endif + + if (result < 0) + { + return errno != 0 ? errno : EINTR; + } + else if (result == 0) + { + return ETIMEDOUT; + } + } + + return 0; +} + +int connectserverbyip(int sock, const char *server_ip, const short server_port) +{ + int result; + struct sockaddr_in addr; + + addr.sin_family = PF_INET; + addr.sin_port = htons(server_port); + result = inet_aton(server_ip, &addr.sin_addr); + if (result == 0 ) + { + return EINVAL; + } + + result = connect(sock, (const struct sockaddr*)&addr, sizeof(addr)); + if (result < 0) + { + return errno != 0 ? errno : EINTR; + } + + return 0; +} + +int connectserverbyip_nb_ex(int sock, const char *server_ip, \ + const short server_port, const int timeout, \ + const bool auto_detect) +{ + int result; + int flags; + bool needRestore; + socklen_t len; + +#ifdef USE_SELECT + fd_set rset; + fd_set wset; + struct timeval tval; +#else + struct pollfd pollfds; +#endif + + struct sockaddr_in addr; + + addr.sin_family = PF_INET; + addr.sin_port = htons(server_port); + result = inet_aton(server_ip, &addr.sin_addr); + if (result == 0 ) + { + return EINVAL; + } + + if (auto_detect) + { + flags = fcntl(sock, F_GETFL, 0); + if (flags < 0) + { + return errno != 0 ? errno : EACCES; + } + + if ((flags & O_NONBLOCK) == 0) + { + if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0) + { + return errno != 0 ? errno : EACCES; + } + + needRestore = true; + } + else + { + needRestore = false; + } + } + else + { + needRestore = false; + flags = 0; + } + + do + { + if (connect(sock, (const struct sockaddr*)&addr, \ + sizeof(addr)) < 0) + { + result = errno != 0 ? errno : EINPROGRESS; + if (result != EINPROGRESS) + { + break; + } + } + else + { + result = 0; + break; + } + + +#ifdef USE_SELECT + FD_ZERO(&rset); + FD_ZERO(&wset); + FD_SET(sock, &rset); + FD_SET(sock, &wset); + tval.tv_sec = timeout; + tval.tv_usec = 0; + + result = select(sock+1, &rset, &wset, NULL, \ + timeout > 0 ? &tval : NULL); +#else + pollfds.fd = sock; + pollfds.events = POLLIN | POLLOUT; + result = poll(&pollfds, 1, 1000 * timeout); +#endif + + if (result == 0) + { + result = ETIMEDOUT; + break; + } + else if (result < 0) + { + result = errno != 0 ? errno : EINTR; + break; + } + + len = sizeof(result); + if (getsockopt(sock, SOL_SOCKET, SO_ERROR, &result, &len) < 0) + { + result = errno != 0 ? errno : EACCES; + break; + } + } while (0); + + if (needRestore) + { + fcntl(sock, F_SETFL, flags); + } + + return result; +} + +in_addr_t getIpaddr(getnamefunc getname, int sock, \ + char *buff, const int bufferSize) +{ + struct sockaddr_in addr; + socklen_t addrlen; + + memset(&addr, 0, sizeof(addr)); + addrlen = sizeof(addr); + + if (getname(sock, (struct sockaddr *)&addr, &addrlen) != 0) + { + *buff = '\0'; + return INADDR_NONE; + } + + if (addrlen > 0) + { + if (inet_ntop(AF_INET, &addr.sin_addr, buff, bufferSize) == NULL) + { + *buff = '\0'; + } + } + else + { + *buff = '\0'; + } + + return addr.sin_addr.s_addr; +} + +char *getHostnameByIp(const char *szIpAddr, char *buff, const int bufferSize) +{ + struct in_addr ip_addr; + struct hostent *ent; + + if (inet_pton(AF_INET, szIpAddr, &ip_addr) != 1) + { + *buff = '\0'; + return buff; + } + + ent = gethostbyaddr((char *)&ip_addr, sizeof(ip_addr), AF_INET); + if (ent == NULL || ent->h_name == NULL) + { + *buff = '\0'; + } + else + { + snprintf(buff, bufferSize, "%s", ent->h_name); + } + + return buff; +} + +in_addr_t getIpaddrByName(const char *name, char *buff, const int bufferSize) +{ + struct in_addr ip_addr; + struct hostent *ent; + in_addr_t **addr_list; + + if ((*name >= '0' && *name <= '9') && + inet_pton(AF_INET, name, &ip_addr) == 1) + { + if (buff != NULL) + { + snprintf(buff, bufferSize, "%s", name); + } + return ip_addr.s_addr; + } + + ent = gethostbyname(name); + if (ent == NULL) + { + return INADDR_NONE; + } + + addr_list = (in_addr_t **)ent->h_addr_list; + if (addr_list[0] == NULL) + { + return INADDR_NONE; + } + + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = *(addr_list[0]); + if (buff != NULL) + { + if (inet_ntop(AF_INET, &ip_addr, buff, bufferSize) == NULL) + { + *buff = '\0'; + } + } + + return ip_addr.s_addr; +} + +int nbaccept(int sock, const int timeout, int *err_no) +{ + struct sockaddr_in inaddr; + socklen_t sockaddr_len; + fd_set read_set; + struct timeval t; + int result; + + if (timeout > 0) + { + t.tv_usec = 0; + t.tv_sec = timeout; + + FD_ZERO(&read_set); + FD_SET(sock, &read_set); + + result = select(sock+1, &read_set, NULL, NULL, &t); + if (result == 0) //timeout + { + *err_no = ETIMEDOUT; + return -1; + } + else if (result < 0) //error + { + *err_no = errno != 0 ? errno : EINTR; + return -1; + } + + /* + if (!FD_ISSET(sock, &read_set)) + { + *err_no = EAGAIN; + return -1; + } + */ + } + + sockaddr_len = sizeof(inaddr); + result = accept(sock, (struct sockaddr*)&inaddr, &sockaddr_len); + if (result < 0) + { + *err_no = errno != 0 ? errno : EINTR; + } + else + { + *err_no = 0; + } + + return result; +} + +int socketBind(int sock, const char *bind_ipaddr, const int port) +{ + struct sockaddr_in bindaddr; + + bindaddr.sin_family = AF_INET; + bindaddr.sin_port = htons(port); + if (bind_ipaddr == NULL || *bind_ipaddr == '\0') + { + bindaddr.sin_addr.s_addr = INADDR_ANY; + } + else + { + if (inet_aton(bind_ipaddr, &bindaddr.sin_addr) == 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid ip addr %s", \ + __LINE__, bind_ipaddr); + return EINVAL; + } + } + + if (bind(sock, (struct sockaddr*)&bindaddr, sizeof(bindaddr)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "bind port %d failed, " \ + "errno: %d, error info: %s.", \ + __LINE__, port, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + return 0; +} + +int socketServer(const char *bind_ipaddr, const int port, int *err_no) +{ + int sock; + int result; + + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) + { + *err_no = errno != 0 ? errno : EMFILE; + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return -1; + } + + result = 1; + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &result, sizeof(int))<0) + { + *err_no = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + close(sock); + return -2; + } + + if ((*err_no=socketBind(sock, bind_ipaddr, port)) != 0) + { + close(sock); + return -3; + } + + if (listen(sock, 1024) < 0) + { + *err_no = errno != 0 ? errno : EINVAL; + logError("file: "__FILE__", line: %d, " \ + "listen port %d failed, " \ + "errno: %d, error info: %s", \ + __LINE__, port, errno, STRERROR(errno)); + close(sock); + return -4; + } + + *err_no = 0; + return sock; +} + +int tcprecvfile(int sock, const char *filename, const int64_t file_bytes, \ + const int fsync_after_written_bytes, const int timeout, \ + int64_t *true_file_bytes) +{ + int write_fd; + char buff[FDFS_WRITE_BUFF_SIZE]; + int64_t remain_bytes; + int recv_bytes; + int written_bytes; + int result; + int flags; + int count; + tcprecvdata_exfunc recv_func; + + *true_file_bytes = 0; + flags = fcntl(sock, F_GETFL, 0); + if (flags < 0) + { + return errno != 0 ? errno : EACCES; + } + + if (flags & O_NONBLOCK) + { + recv_func = tcprecvdata_nb_ex; + } + else + { + recv_func = tcprecvdata_ex; + } + + write_fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (write_fd < 0) + { + return errno != 0 ? errno : EACCES; + } + + written_bytes = 0; + remain_bytes = file_bytes; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + recv_bytes = sizeof(buff); + } + else + { + recv_bytes = remain_bytes; + } + + result = recv_func(sock, buff, recv_bytes, \ + timeout, &count); + if (result != 0) + { + if (file_bytes != INFINITE_FILE_SIZE) + { + close(write_fd); + unlink(filename); + return result; + } + } + + if (count > 0 && write(write_fd, buff, count) != count) + { + result = errno != 0 ? errno: EIO; + close(write_fd); + unlink(filename); + return result; + } + + *true_file_bytes += count; + if (fsync_after_written_bytes > 0) + { + written_bytes += count; + if (written_bytes >= fsync_after_written_bytes) + { + written_bytes = 0; + if (fsync(write_fd) != 0) + { + result = errno != 0 ? errno: EIO; + close(write_fd); + unlink(filename); + return result; + } + } + } + + if (result != 0) //recv infinite file, does not delete the file + { + int read_fd; + read_fd = -1; + + do + { + if (*true_file_bytes < 8) + { + break; + } + + read_fd = open(filename, O_RDONLY); + if (read_fd < 0) + { + return errno != 0 ? errno : EACCES; + } + + if (lseek(read_fd, -8, SEEK_END) < 0) + { + result = errno != 0 ? errno : EIO; + break; + } + + if (read(read_fd, buff, 8) != 8) + { + result = errno != 0 ? errno : EIO; + break; + } + + *true_file_bytes -= 8; + if (buff2long(buff) != *true_file_bytes) + { + result = EINVAL; + break; + } + + if (ftruncate(write_fd, *true_file_bytes) != 0) + { + result = errno != 0 ? errno : EIO; + break; + } + + result = 0; + } while (0); + + close(write_fd); + if (read_fd >= 0) + { + close(read_fd); + } + + if (result != 0) + { + unlink(filename); + } + + return result; + } + + remain_bytes -= count; + } + + close(write_fd); + return 0; +} + +int tcprecvfile_ex(int sock, const char *filename, const int64_t file_bytes, \ + const int fsync_after_written_bytes, \ + unsigned int *hash_codes, const int timeout) +{ + int fd; + char buff[FDFS_WRITE_BUFF_SIZE]; + int64_t remain_bytes; + int recv_bytes; + int written_bytes; + int result; + int flags; + tcprecvdata_exfunc recv_func; + + flags = fcntl(sock, F_GETFL, 0); + if (flags < 0) + { + return errno != 0 ? errno : EACCES; + } + + if (flags & O_NONBLOCK) + { + recv_func = tcprecvdata_nb_ex; + } + else + { + recv_func = tcprecvdata_ex; + } + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + { + return errno != 0 ? errno : EACCES; + } + + INIT_HASH_CODES4(hash_codes) + + written_bytes = 0; + remain_bytes = file_bytes; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + recv_bytes = sizeof(buff); + } + else + { + recv_bytes = remain_bytes; + } + + if ((result=recv_func(sock, buff, recv_bytes, \ + timeout, NULL)) != 0) + { + close(fd); + unlink(filename); + return result; + } + + if (write(fd, buff, recv_bytes) != recv_bytes) + { + result = errno != 0 ? errno: EIO; + close(fd); + unlink(filename); + return result; + } + + if (fsync_after_written_bytes > 0) + { + written_bytes += recv_bytes; + if (written_bytes >= fsync_after_written_bytes) + { + written_bytes = 0; + if (fsync(fd) != 0) + { + result = errno != 0 ? errno: EIO; + close(fd); + unlink(filename); + return result; + } + } + } + + CALC_HASH_CODES4(buff, recv_bytes, hash_codes) + + remain_bytes -= recv_bytes; + } + + close(fd); + + FINISH_HASH_CODES4(hash_codes) + + return 0; +} + +int tcpdiscard(int sock, const int bytes, const int timeout, \ + int64_t *total_recv_bytes) +{ + char buff[FDFS_WRITE_BUFF_SIZE]; + int remain_bytes; + int recv_bytes; + int result; + int flags; + int count; + tcprecvdata_exfunc recv_func; + + *total_recv_bytes = 0; + flags = fcntl(sock, F_GETFL, 0); + if (flags < 0) + { + return errno != 0 ? errno : EACCES; + } + + if (flags & O_NONBLOCK) + { + recv_func = tcprecvdata_nb_ex; + } + else + { + recv_func = tcprecvdata_ex; + } + + remain_bytes = bytes; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + recv_bytes = sizeof(buff); + } + else + { + recv_bytes = remain_bytes; + } + + result = recv_func(sock, buff, recv_bytes, \ + timeout, &count); + *total_recv_bytes += count; + if (result != 0) + { + return result; + } + + remain_bytes -= recv_bytes; + } + + return 0; +} + +int tcpsendfile_ex(int sock, const char *filename, const int64_t file_offset, \ + const int64_t file_bytes, const int timeout, int64_t *total_send_bytes) +{ + int fd; + int64_t send_bytes; + int result; + int flags; +#ifdef USE_SENDFILE + #if defined(OS_FREEBSD) || defined(OS_LINUX) + off_t offset; + #ifdef OS_LINUX + int64_t remain_bytes; + #endif + #endif +#else + int64_t remain_bytes; +#endif + + fd = open(filename, O_RDONLY); + if (fd < 0) + { + *total_send_bytes = 0; + return errno != 0 ? errno : EACCES; + } + + flags = fcntl(sock, F_GETFL, 0); + if (flags < 0) + { + *total_send_bytes = 0; + return errno != 0 ? errno : EACCES; + } + +#ifdef USE_SENDFILE + + if (flags & O_NONBLOCK) + { + if (fcntl(sock, F_SETFL, flags & ~O_NONBLOCK) == -1) + { + *total_send_bytes = 0; + return errno != 0 ? errno : EACCES; + } + } + +#ifdef OS_LINUX + /* + result = 1; + if (setsockopt(sock, SOL_TCP, TCP_CORK, &result, sizeof(int)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + close(fd); + *total_send_bytes = 0; + return errno != 0 ? errno : EIO; + } + */ + +#define FILE_1G_SIZE (1 * 1024 * 1024 * 1024) + + result = 0; + offset = file_offset; + remain_bytes = file_bytes; + while (remain_bytes > 0) + { + if (remain_bytes > FILE_1G_SIZE) + { + send_bytes = sendfile(sock, fd, &offset, FILE_1G_SIZE); + } + else + { + send_bytes = sendfile(sock, fd, &offset, remain_bytes); + } + + if (send_bytes <= 0) + { + result = errno != 0 ? errno : EIO; + break; + } + + remain_bytes -= send_bytes; + } + + *total_send_bytes = file_bytes - remain_bytes; +#else +#ifdef OS_FREEBSD + offset = file_offset; + if (sendfile(fd, sock, offset, file_bytes, NULL, NULL, 0) != 0) + { + *total_send_bytes = 0; + result = errno != 0 ? errno : EIO; + } + else + { + *total_send_bytes = file_bytes; + result = 0; + } +#endif +#endif + + if (flags & O_NONBLOCK) //restore + { + if (fcntl(sock, F_SETFL, flags) == -1) + { + result = errno != 0 ? errno : EACCES; + } + } + +#ifdef OS_LINUX + close(fd); + return result; +#endif + +#ifdef OS_FREEBSD + close(fd); + return result; +#endif + +#endif + + { + char buff[FDFS_WRITE_BUFF_SIZE]; + int64_t remain_bytes; + tcpsenddatafunc send_func; + + if (file_offset > 0 && lseek(fd, file_offset, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + close(fd); + *total_send_bytes = 0; + return result; + } + + if (flags & O_NONBLOCK) + { + send_func = tcpsenddata_nb; + } + else + { + send_func = tcpsenddata; + } + + result = 0; + remain_bytes = file_bytes; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + send_bytes = sizeof(buff); + } + else + { + send_bytes = remain_bytes; + } + + if (read(fd, buff, send_bytes) != send_bytes) + { + result = errno != 0 ? errno : EIO; + break; + } + + if ((result=send_func(sock, buff, send_bytes, \ + timeout)) != 0) + { + break; + } + + remain_bytes -= send_bytes; + } + + *total_send_bytes = file_bytes - remain_bytes; + } + + close(fd); + return result; +} + +int tcpsetserveropt(int fd, const int timeout) +{ + int flags; + int result; + + struct linger linger; + struct timeval waittime; + +/* + linger.l_onoff = 1; +#ifdef OS_FREEBSD + linger.l_linger = timeout * 100; +#else + linger.l_linger = timeout; +#endif +*/ + linger.l_onoff = 0; + linger.l_linger = 0; + if (setsockopt(fd, SOL_SOCKET, SO_LINGER, \ + &linger, (socklen_t)sizeof(struct linger)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + waittime.tv_sec = timeout; + waittime.tv_usec = 0; + if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, + &waittime, (socklen_t)sizeof(struct timeval)) < 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, + &waittime, (socklen_t)sizeof(struct timeval)) < 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + + flags = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, \ + (char *)&flags, sizeof(flags)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + if ((result=tcpsetkeepalive(fd, 2 * timeout + 1)) != 0) + { + return result; + } + + return 0; +} + +int tcpsetkeepalive(int fd, const int idleSeconds) +{ + int keepAlive; + +#ifdef OS_LINUX + int keepIdle; + int keepInterval; + int keepCount; +#endif + + keepAlive = 1; + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, \ + (char *)&keepAlive, sizeof(keepAlive)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + +#ifdef OS_LINUX + keepIdle = idleSeconds; + if (setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, (char *)&keepIdle, \ + sizeof(keepIdle)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + keepInterval = 10; + if (setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, (char *)&keepInterval, \ + sizeof(keepInterval)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + keepCount = 3; + if (setsockopt(fd, SOL_TCP, TCP_KEEPCNT, (char *)&keepCount, \ + sizeof(keepCount)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } +#endif + + return 0; +} + +int tcpprintkeepalive(int fd) +{ + int keepAlive; + socklen_t len; + +#ifdef OS_LINUX + int keepIdle; + int keepInterval; + int keepCount; +#endif + + len = sizeof(keepAlive); + if (getsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, \ + (char *)&keepAlive, &len) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + +#ifdef OS_LINUX + len = sizeof(keepIdle); + if (getsockopt(fd, SOL_TCP, TCP_KEEPIDLE, (char *)&keepIdle, \ + &len) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + len = sizeof(keepInterval); + if (getsockopt(fd, SOL_TCP, TCP_KEEPINTVL, (char *)&keepInterval, \ + &len) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + len = sizeof(keepCount); + if (getsockopt(fd, SOL_TCP, TCP_KEEPCNT, (char *)&keepCount, \ + &len) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + logInfo("keepAlive=%d, keepIdle=%d, keepInterval=%d, keepCount=%d", + keepAlive, keepIdle, keepInterval, keepCount); +#else + logInfo("keepAlive=%d", keepAlive); +#endif + + return 0; +} + +int tcpsetnonblockopt(int fd) +{ + int flags; + + flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) + { + logError("file: "__FILE__", line: %d, " \ + "fcntl failed, errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) + { + logError("file: "__FILE__", line: %d, " \ + "fcntl failed, errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +int tcpsetnodelay(int fd, const int timeout) +{ + int flags; + int result; + + if ((result=tcpsetkeepalive(fd, 2 * timeout + 1)) != 0) + { + return result; + } + + flags = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, \ + (char *)&flags, sizeof(flags)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "setsockopt failed, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EINVAL; + } + + return 0; +} + +#if defined(OS_LINUX) || defined(OS_FREEBSD) +int getlocaladdrs(char ip_addrs[][IP_ADDRESS_SIZE], \ + const int max_count, int *count) +{ + struct ifaddrs *ifc; + struct ifaddrs *ifc1; + + *count = 0; + if (0 != getifaddrs(&ifc)) + { + logError("file: "__FILE__", line: %d, " \ + "call getifaddrs fail, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EMFILE; + } + + ifc1 = ifc; + while (NULL != ifc) + { + struct sockaddr *s; + s = ifc->ifa_addr; + if (NULL != s && AF_INET == s->sa_family) + { + if (max_count <= *count) + { + logError("file: "__FILE__", line: %d, "\ + "max_count: %d < iterface count: %d", \ + __LINE__, max_count, *count); + freeifaddrs(ifc1); + return ENOSPC; + } + + if (inet_ntop(AF_INET, &((struct sockaddr_in *)s)-> \ + sin_addr, ip_addrs[*count], IP_ADDRESS_SIZE) != NULL) + { + (*count)++; + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "call inet_ntop fail, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + } + + ifc = ifc->ifa_next; + } + + freeifaddrs(ifc1); + return *count > 0 ? 0 : ENOENT; +} + +#else + +int getlocaladdrs(char ip_addrs[][IP_ADDRESS_SIZE], \ + const int max_count, int *count) +{ + int s; + struct ifconf ifconf; + struct ifreq ifr[32]; + int if_count; + int i; + int result; + + *count = 0; + s = socket(AF_INET, SOCK_STREAM, 0); + if (s < 0) + { + logError("file: "__FILE__", line: %d, " \ + "socket create fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EMFILE; + } + + ifconf.ifc_buf = (char *) ifr; + ifconf.ifc_len = sizeof(ifr); + if (ioctl(s, SIOCGIFCONF, &ifconf) < 0) + { + result = errno != 0 ? errno : EMFILE; + logError("file: "__FILE__", line: %d, " \ + "call ioctl fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + close(s); + return result; + } + + if_count = ifconf.ifc_len / sizeof(ifr[0]); + if (max_count < if_count) + { + logError("file: "__FILE__", line: %d, " \ + "max_count: %d < iterface count: %d", \ + __LINE__, max_count, if_count); + close(s); + return ENOSPC; + } + + for (i = 0; i < if_count; i++) + { + struct sockaddr_in *s_in; + s_in = (struct sockaddr_in *) &ifr[i].ifr_addr; + if (!inet_ntop(AF_INET, &s_in->sin_addr, \ + ip_addrs[*count], IP_ADDRESS_SIZE)) + { + result = errno != 0 ? errno : EMFILE; + logError("file: "__FILE__", line: %d, " \ + "call inet_ntop fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + close(s); + return result; + } + (*count)++; + } + + close(s); + return *count > 0 ? 0 : ENOENT; +} + +#endif + +int gethostaddrs(char **if_alias_prefixes, const int prefix_count, \ + char ip_addrs[][IP_ADDRESS_SIZE], const int max_count, int *count) +{ + struct hostent *ent; + char hostname[128]; + char *alias_prefixes1[1]; + char **true_alias_prefixes; + int true_count; + int i; + int k; + int sock; + struct ifreq req; + struct sockaddr_in *addr; + int ret; + + *count = 0; + if (prefix_count <= 0) + { + if (getlocaladdrs(ip_addrs, max_count, count) == 0) + { + return 0; + } + +#ifdef OS_FREEBSD + #define IF_NAME_PREFIX "bge" +#else + #ifdef OS_SUNOS + #define IF_NAME_PREFIX "e1000g" + #else + #ifdef OS_AIX + #define IF_NAME_PREFIX "en" + #else + #define IF_NAME_PREFIX "eth" + #endif + #endif +#endif + + alias_prefixes1[0] = IF_NAME_PREFIX; + true_count = 1; + true_alias_prefixes = alias_prefixes1; + } + else + { + true_count = prefix_count; + true_alias_prefixes = if_alias_prefixes; + } + + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, error info: %s.", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EMFILE; + } + + for (i=0; isin_addr, ip_addrs[*count], \ + IP_ADDRESS_SIZE) != NULL) + { + (*count)++; + if (*count >= max_count) + { + break; + } + } + } + } + + close(sock); + if (*count > 0) + { + return 0; + } + + if (gethostname(hostname, sizeof(hostname)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call gethostname fail, " \ + "error no: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + return errno != 0 ? errno : EFAULT; + } + + ent = gethostbyname(hostname); + if (ent == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "call gethostbyname fail, " \ + "error no: %d, error info: %s", \ + __LINE__, h_errno, STRERROR(h_errno)); + return h_errno != 0 ? h_errno : EFAULT; + } + + k = 0; + while (ent->h_addr_list[k] != NULL) + { + if (*count >= max_count) + { + break; + } + + if (inet_ntop(ent->h_addrtype, ent->h_addr_list[k], \ + ip_addrs[*count], IP_ADDRESS_SIZE) != NULL) + { + (*count)++; + } + + k++; + } + + return 0; +} + diff --git a/common/sockopt.h b/common/sockopt.h new file mode 100644 index 0000000..1c8773c --- /dev/null +++ b/common/sockopt.h @@ -0,0 +1,335 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//socketopt.h + +#ifndef _SOCKETOPT_H_ +#define _SOCKETOPT_H_ + +#include "common_define.h" + +#define FDFS_WRITE_BUFF_SIZE 256 * 1024 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int (*getnamefunc)(int socket, struct sockaddr *address, \ + socklen_t *address_len); + +typedef int (*tcpsenddatafunc)(int sock, void* data, const int size, \ + const int timeout); + +typedef int (*tcprecvdata_exfunc)(int sock, void *data, const int size, \ + const int timeout, int *count); + +#define getSockIpaddr(sock, buff, bufferSize) \ + getIpaddr(getsockname, sock, buff, bufferSize) + +#define getPeerIpaddr(sock, buff, bufferSize) \ + getIpaddr(getpeername, sock, buff, bufferSize) + +/** get a line from socket + * parameters: + * sock: the socket + * s: the buffer + * size: buffer size (max bytes can receive) + * timeout: read timeout + * return: error no, 0 success, != 0 fail +*/ +int tcpgets(int sock, char *s, const int size, const int timeout); + +/** recv data (block mode) + * parameters: + * sock: the socket + * data: the buffer + * size: buffer size (max bytes can receive) + * timeout: read timeout + * count: store the bytes recveived + * return: error no, 0 success, != 0 fail +*/ +int tcprecvdata_ex(int sock, void *data, const int size, \ + const int timeout, int *count); + +/** recv data (non-block mode) + * parameters: + * sock: the socket + * data: the buffer + * size: buffer size (max bytes can receive) + * timeout: read timeout + * count: store the bytes recveived + * return: error no, 0 success, != 0 fail +*/ +int tcprecvdata_nb_ex(int sock, void *data, const int size, \ + const int timeout, int *count); + +/** send data (block mode) + * parameters: + * sock: the socket + * data: the buffer to send + * size: buffer size + * timeout: write timeout + * return: error no, 0 success, != 0 fail +*/ +int tcpsenddata(int sock, void* data, const int size, const int timeout); + +/** send data (non-block mode) + * parameters: + * sock: the socket + * data: the buffer to send + * size: buffer size + * timeout: write timeout + * return: error no, 0 success, != 0 fail +*/ +int tcpsenddata_nb(int sock, void* data, const int size, const int timeout); + +/** connect to server by block mode + * parameters: + * sock: the socket + * server_ip: ip address of the server + * server_port: port of the server + * return: error no, 0 success, != 0 fail +*/ +int connectserverbyip(int sock, const char *server_ip, const short server_port); + +/** connect to server by non-block mode + * parameters: + * sock: the socket + * server_ip: ip address of the server + * server_port: port of the server + * timeout: connect timeout in seconds + * auto_detect: if detect and adjust the block mode of the socket + * return: error no, 0 success, != 0 fail +*/ +int connectserverbyip_nb_ex(int sock, const char *server_ip, \ + const short server_port, const int timeout, \ + const bool auto_detect); + +/** connect to server by non-block mode, the socket must be set to non-block + * parameters: + * sock: the socket, must be set to non-block + * server_ip: ip address of the server + * server_port: port of the server + * timeout: connect timeout in seconds + * return: error no, 0 success, != 0 fail +*/ +#define connectserverbyip_nb(sock, server_ip, server_port, timeout) \ + connectserverbyip_nb_ex(sock, server_ip, server_port, timeout, false) + +/** connect to server by non-block mode, auto detect socket block mode + * parameters: + * sock: the socket, can be block mode + * server_ip: ip address of the server + * server_port: port of the server + * timeout: connect timeout in seconds + * return: error no, 0 success, != 0 fail +*/ +#define connectserverbyip_nb_auto(sock, server_ip, server_port, timeout) \ + connectserverbyip_nb_ex(sock, server_ip, server_port, timeout, true) + +/** accept client connect request + * parameters: + * sock: the server socket + * timeout: read timeout + * err_no: store the error no, 0 for success + * return: client socket, < 0 for error +*/ +int nbaccept(int sock, const int timeout, int *err_no); + +/** set socket options + * parameters: + * sock: the socket + * timeout: read & write timeout + * return: error no, 0 success, != 0 fail +*/ +int tcpsetserveropt(int fd, const int timeout); + +/** set socket non-block options + * parameters: + * sock: the socket + * return: error no, 0 success, != 0 fail +*/ +int tcpsetnonblockopt(int fd); + +/** set socket no delay on send data + * parameters: + * sock: the socket + * timeout: read & write timeout + * return: error no, 0 success, != 0 fail +*/ +int tcpsetnodelay(int fd, const int timeout); + +/** set socket keep-alive + * parameters: + * sock: the socket + * idleSeconds: max idle time (seconds) + * return: error no, 0 success, != 0 fail +*/ +int tcpsetkeepalive(int fd, const int idleSeconds); + +/** print keep-alive related parameters + * parameters: + * sock: the socket + * return: error no, 0 success, != 0 fail +*/ +int tcpprintkeepalive(int fd); + +/** get ip address + * parameters: + * getname: the function name, should be getpeername or getsockname + * sock: the socket + * buff: buffer to store the ip address + * bufferSize: the buffer size (max bytes) + * return: in_addr_t, INADDR_NONE for fail +*/ +in_addr_t getIpaddr(getnamefunc getname, int sock, \ + char *buff, const int bufferSize); + +/** get hostname by it's ip address + * parameters: + * szIpAddr: the ip address + * buff: buffer to store the hostname + * bufferSize: the buffer size (max bytes) + * return: hostname, empty buffer for error +*/ +char *getHostnameByIp(const char *szIpAddr, char *buff, const int bufferSize); + +/** get by ip address by it's hostname + * parameters: + * name: the hostname + * buff: buffer to store the ip address + * bufferSize: the buffer size (max bytes) + * return: in_addr_t, INADDR_NONE for fail +*/ +in_addr_t getIpaddrByName(const char *name, char *buff, const int bufferSize); + +/** bind wrapper + * parameters: + * sock: the socket + * bind_ipaddr: the ip address to bind + * port: the port to bind + * return: error no, 0 success, != 0 fail +*/ +int socketBind(int sock, const char *bind_ipaddr, const int port); + +/** start a socket server (socket, bind and listen) + * parameters: + * sock: the socket + * bind_ipaddr: the ip address to bind + * port: the port to bind + * err_no: store the error no + * return: >= 0 server socket, < 0 fail +*/ +int socketServer(const char *bind_ipaddr, const int port, int *err_no); + +#define tcprecvdata(sock, data, size, timeout) \ + tcprecvdata_ex(sock, data, size, timeout, NULL) + +#define tcpsendfile(sock, filename, file_bytes, timeout, total_send_bytes) \ + tcpsendfile_ex(sock, filename, 0, file_bytes, timeout, total_send_bytes) + +#define tcprecvdata_nb(sock, data, size, timeout) \ + tcprecvdata_nb_ex(sock, data, size, timeout, NULL) + +/** send a file + * parameters: + * sock: the socket + * filename: the file to send + * file_offset: file offset, start position + * file_bytes: send file length + * timeout: write timeout + * total_send_bytes: store the send bytes + * return: error no, 0 success, != 0 fail +*/ +int tcpsendfile_ex(int sock, const char *filename, const int64_t file_offset, \ + const int64_t file_bytes, const int timeout, int64_t *total_send_bytes); + +/** receive data to a file + * parameters: + * sock: the socket + * filename: the file to write + * file_bytes: file size (bytes) + * fsync_after_written_bytes: call fsync every x bytes + * timeout: read/recv timeout + * true_file_bytes: store the true file bytes + * return: error no, 0 success, != 0 fail +*/ +int tcprecvfile(int sock, const char *filename, const int64_t file_bytes, \ + const int fsync_after_written_bytes, const int timeout, \ + int64_t *true_file_bytes); + + +#define tcprecvinfinitefile(sock, filename, fsync_after_written_bytes, \ + timeout, file_bytes) \ + tcprecvfile(sock, filename, INFINITE_FILE_SIZE, \ + fsync_after_written_bytes, timeout, file_bytes) + + +/** receive data to a file + * parameters: + * sock: the socket + * filename: the file to write + * file_bytes: file size (bytes) + * fsync_after_written_bytes: call fsync every x bytes + * hash_codes: return hash code of file content + * timeout: read/recv timeout + * return: error no, 0 success, != 0 fail +*/ +int tcprecvfile_ex(int sock, const char *filename, const int64_t file_bytes, \ + const int fsync_after_written_bytes, \ + unsigned int *hash_codes, const int timeout); + +/** receive specified data and discard + * parameters: + * sock: the socket + * bytes: data bytes to discard + * timeout: read timeout + * total_recv_bytes: store the total recv bytes + * return: error no, 0 success, != 0 fail +*/ +int tcpdiscard(int sock, const int bytes, const int timeout, \ + int64_t *total_recv_bytes); + +/** get local host ip addresses + * parameters: + * ip_addrs: store the ip addresses + * max_count: max ip address (max ip_addrs elements) + * count: store the ip address count + * return: error no, 0 success, != 0 fail +*/ +int getlocaladdrs(char ip_addrs[][IP_ADDRESS_SIZE], \ + const int max_count, int *count); + +/** get local host ip addresses + * parameters: + * ip_addrs: store the ip addresses + * max_count: max ip address (max ip_addrs elements) + * count: store the ip address count + * return: error no, 0 success, != 0 fail +*/ +int getlocaladdrs1(char ip_addrs[][IP_ADDRESS_SIZE], \ + const int max_count, int *count); + +/** get local host ip addresses by if alias prefix + * parameters: + * if_alias_prefixes: if alias prefixes, such as eth, bond etc. + * prefix_count: if alias prefix count + * ip_addrs: store the ip addresses + * max_count: max ip address (max ip_addrs elements) + * count: store the ip address count + * return: error no, 0 success, != 0 fail +*/ +int gethostaddrs(char **if_alias_prefixes, const int prefix_count, \ + char ip_addrs[][IP_ADDRESS_SIZE], const int max_count, int *count); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/conf/anti-steal.jpg b/conf/anti-steal.jpg new file mode 100644 index 0000000..a52584b Binary files /dev/null and b/conf/anti-steal.jpg differ diff --git a/conf/client.conf b/conf/client.conf new file mode 100644 index 0000000..f935743 --- /dev/null +++ b/conf/client.conf @@ -0,0 +1,62 @@ +# connect timeout in seconds +# default value is 30s +connect_timeout=30 + +# network timeout in seconds +# default value is 30s +network_timeout=60 + +# the base path to store log files +base_path=/home/yuqing/fastdfs + +# tracker_server can ocur more than once, and tracker_server format is +# "host:port", host can be hostname or ip address +tracker_server=192.168.0.197:22122 + +#standard log level as syslog, case insensitive, value list: +### emerg for emergency +### alert +### crit for critical +### error +### warn for warning +### notice +### info +### debug +log_level=info + +# if use connection pool +# default value is false +# since V4.05 +use_connection_pool = false + +# connections whose the idle time exceeds this time will be closed +# unit: second +# default value is 3600 +# since V4.05 +connection_pool_max_idle_time = 3600 + +# if load FastDFS parameters from tracker server +# since V4.05 +# default value is false +load_fdfs_parameters_from_tracker=false + +# if use storage ID instead of IP address +# same as tracker.conf +# valid only when load_fdfs_parameters_from_tracker is false +# default value is false +# since V4.05 +use_storage_id = false + +# specify storage ids filename, can use relative or absolute path +# same as tracker.conf +# valid only when load_fdfs_parameters_from_tracker is false +# since V4.05 +storage_ids_filename = storage_ids.conf + + +#HTTP settings +http.tracker_server_port=80 + +#use "#include" directive to include HTTP other settiongs +##include http.conf + diff --git a/conf/http.conf b/conf/http.conf new file mode 100644 index 0000000..6f643de --- /dev/null +++ b/conf/http.conf @@ -0,0 +1,26 @@ +# HTTP default content type +http.default_content_type = application/octet-stream + +# MIME types mapping filename +# MIME types file format: MIME_type extensions +# such as: image/jpeg jpeg jpg jpe +# you can use apache's MIME file: mime.types +http.mime_types_filename=mime.types + +# if use token to anti-steal +# default value is false (0) +http.anti_steal.check_token=false + +# token TTL (time to live), seconds +# default value is 600 +http.anti_steal.token_ttl=900 + +# secret key to generate anti-steal token +# this parameter must be set when http.anti_steal.check_token set to true +# the length of the secret key should not exceed 128 bytes +http.anti_steal.secret_key=FastDFS1234567890 + +# return the content of the file when check token fail +# default value is empty (no file sepecified) +http.anti_steal.token_check_fail=/home/yuqing/fastdfs/conf/anti-steal.jpg + diff --git a/conf/mime.types b/conf/mime.types new file mode 100644 index 0000000..07b4465 --- /dev/null +++ b/conf/mime.types @@ -0,0 +1,1065 @@ +# This is a comment. I love comments. + +# This file controls what Internet media types are sent to the client for +# given file extension(s). Sending the correct media type to the client +# is important so they know how to handle the content of the file. +# Extra types can either be added here or by using an AddType directive +# in your config files. For more information about Internet media types, +# please read RFC 2045, 2046, 2047, 2048, and 2077. The Internet media type +# registry is at . + +# MIME type Extensions +application/activemessage +application/andrew-inset ez +application/applefile +application/atom+xml atom +application/atomcat+xml atomcat +application/atomicmail +application/atomsvc+xml atomsvc +application/auth-policy+xml +application/batch-smtp +application/beep+xml +application/cals-1840 +application/ccxml+xml ccxml +application/cellml+xml +application/cnrp+xml +application/commonground +application/conference-info+xml +application/cpl+xml +application/csta+xml +application/cstadata+xml +application/cybercash +application/davmount+xml davmount +application/dca-rft +application/dec-dx +application/dialog-info+xml +application/dicom +application/dns +application/dvcs +application/ecmascript ecma +application/edi-consent +application/edi-x12 +application/edifact +application/epp+xml +application/eshop +application/fastinfoset +application/fastsoap +application/fits +application/font-tdpfr pfr +application/h224 +application/http +application/hyperstudio stk +application/iges +application/im-iscomposing+xml +application/index +application/index.cmd +application/index.obj +application/index.response +application/index.vnd +application/iotp +application/ipp +application/isup +application/javascript js +application/json json +application/kpml-request+xml +application/kpml-response+xml +application/lost+xml lostxml +application/mac-binhex40 hqx +application/mac-compactpro cpt +application/macwriteii +application/marc mrc +application/mathematica ma nb mb +application/mathml+xml mathml +application/mbms-associated-procedure-description+xml +application/mbms-deregister+xml +application/mbms-envelope+xml +application/mbms-msk+xml +application/mbms-msk-response+xml +application/mbms-protection-description+xml +application/mbms-reception-report+xml +application/mbms-register+xml +application/mbms-register-response+xml +application/mbms-user-service-description+xml +application/mbox mbox +application/media_control+xml +application/mediaservercontrol+xml mscml +application/mikey +application/moss-keys +application/moss-signature +application/mosskey-data +application/mosskey-request +application/mp4 mp4s +application/mpeg4-generic +application/mpeg4-iod +application/mpeg4-iod-xmt +application/msword doc dot +application/mxf mxf +application/nasdata +application/news-transmission +application/nss +application/ocsp-request +application/ocsp-response +application/octet-stream bin dms lha lzh class so iso dmg dist distz pkg bpk dump elc +application/oda oda +application/oebps-package+xml +application/ogg ogx +application/parityfec +application/patch-ops-error+xml xer +application/pdf pdf +application/pgp-encrypted pgp +application/pgp-keys +application/pgp-signature asc sig +application/pics-rules prf +application/pidf+xml +application/pidf-diff+xml +application/pkcs10 p10 +application/pkcs7-mime p7m p7c +application/pkcs7-signature p7s +application/pkix-cert cer +application/pkix-crl crl +application/pkix-pkipath pkipath +application/pkixcmp pki +application/pls+xml pls +application/poc-settings+xml +application/postscript ai eps ps +application/prs.alvestrand.titrax-sheet +application/prs.cww cww +application/prs.nprend +application/prs.plucker +application/qsig +application/rdf+xml rdf +application/reginfo+xml rif +application/relax-ng-compact-syntax rnc +application/remote-printing +application/resource-lists+xml rl +application/resource-lists-diff+xml rld +application/riscos +application/rlmi+xml +application/rls-services+xml rs +application/rsd+xml rsd +application/rss+xml rss +application/rtf rtf +application/rtx +application/samlassertion+xml +application/samlmetadata+xml +application/sbml+xml sbml +application/scvp-cv-request scq +application/scvp-cv-response scs +application/scvp-vp-request spq +application/scvp-vp-response spp +application/sdp sdp +application/set-payment +application/set-payment-initiation setpay +application/set-registration +application/set-registration-initiation setreg +application/sgml +application/sgml-open-catalog +application/shf+xml shf +application/sieve +application/simple-filter+xml +application/simple-message-summary +application/simplesymbolcontainer +application/slate +application/smil +application/smil+xml smi smil +application/soap+fastinfoset +application/soap+xml +application/sparql-query rq +application/sparql-results+xml srx +application/spirits-event+xml +application/srgs gram +application/srgs+xml grxml +application/ssml+xml ssml +application/timestamp-query +application/timestamp-reply +application/tve-trigger +application/ulpfec +application/vemmi +application/vividence.scriptfile +application/vnd.3gpp.bsf+xml +application/vnd.3gpp.pic-bw-large plb +application/vnd.3gpp.pic-bw-small psb +application/vnd.3gpp.pic-bw-var pvb +application/vnd.3gpp.sms +application/vnd.3gpp2.bcmcsinfo+xml +application/vnd.3gpp2.sms +application/vnd.3gpp2.tcap tcap +application/vnd.3m.post-it-notes pwn +application/vnd.accpac.simply.aso aso +application/vnd.accpac.simply.imp imp +application/vnd.acucobol acu +application/vnd.acucorp atc acutc +application/vnd.adobe.xdp+xml xdp +application/vnd.adobe.xfdf xfdf +application/vnd.aether.imp +application/vnd.americandynamics.acc acc +application/vnd.amiga.ami ami +application/vnd.anser-web-certificate-issue-initiation cii +application/vnd.anser-web-funds-transfer-initiation fti +application/vnd.antix.game-component atx +application/vnd.apple.installer+xml mpkg +application/vnd.arastra.swi swi +application/vnd.audiograph aep +application/vnd.autopackage +application/vnd.avistar+xml +application/vnd.blueice.multipass mpm +application/vnd.bmi bmi +application/vnd.businessobjects rep +application/vnd.cab-jscript +application/vnd.canon-cpdl +application/vnd.canon-lips +application/vnd.cendio.thinlinc.clientconf +application/vnd.chemdraw+xml cdxml +application/vnd.chipnuts.karaoke-mmd mmd +application/vnd.cinderella cdy +application/vnd.cirpack.isdn-ext +application/vnd.claymore cla +application/vnd.clonk.c4group c4g c4d c4f c4p c4u +application/vnd.commerce-battelle +application/vnd.commonspace csp cst +application/vnd.contact.cmsg cdbcmsg +application/vnd.cosmocaller cmc +application/vnd.crick.clicker clkx +application/vnd.crick.clicker.keyboard clkk +application/vnd.crick.clicker.palette clkp +application/vnd.crick.clicker.template clkt +application/vnd.crick.clicker.wordbank clkw +application/vnd.criticaltools.wbs+xml wbs +application/vnd.ctc-posml pml +application/vnd.ctct.ws+xml +application/vnd.cups-pdf +application/vnd.cups-postscript +application/vnd.cups-ppd ppd +application/vnd.cups-raster +application/vnd.cups-raw +application/vnd.curl curl +application/vnd.cybank +application/vnd.data-vision.rdz rdz +application/vnd.denovo.fcselayout-link fe_launch +application/vnd.dna dna +application/vnd.dolby.mlp mlp +application/vnd.dpgraph dpg +application/vnd.dreamfactory dfac +application/vnd.dvb.esgcontainer +application/vnd.dvb.ipdcesgaccess +application/vnd.dvb.iptv.alfec-base +application/vnd.dvb.iptv.alfec-enhancement +application/vnd.dxr +application/vnd.ecdis-update +application/vnd.ecowin.chart mag +application/vnd.ecowin.filerequest +application/vnd.ecowin.fileupdate +application/vnd.ecowin.series +application/vnd.ecowin.seriesrequest +application/vnd.ecowin.seriesupdate +application/vnd.enliven nml +application/vnd.epson.esf esf +application/vnd.epson.msf msf +application/vnd.epson.quickanime qam +application/vnd.epson.salt slt +application/vnd.epson.ssf ssf +application/vnd.ericsson.quickcall +application/vnd.eszigno3+xml es3 et3 +application/vnd.eudora.data +application/vnd.ezpix-album ez2 +application/vnd.ezpix-package ez3 +application/vnd.fdf fdf +application/vnd.ffsns +application/vnd.fints +application/vnd.flographit gph +application/vnd.fluxtime.clip ftc +application/vnd.font-fontforge-sfd +application/vnd.framemaker fm frame maker +application/vnd.frogans.fnc fnc +application/vnd.frogans.ltf ltf +application/vnd.fsc.weblaunch fsc +application/vnd.fujitsu.oasys oas +application/vnd.fujitsu.oasys2 oa2 +application/vnd.fujitsu.oasys3 oa3 +application/vnd.fujitsu.oasysgp fg5 +application/vnd.fujitsu.oasysprs bh2 +application/vnd.fujixerox.art-ex +application/vnd.fujixerox.art4 +application/vnd.fujixerox.hbpl +application/vnd.fujixerox.ddd ddd +application/vnd.fujixerox.docuworks xdw +application/vnd.fujixerox.docuworks.binder xbd +application/vnd.fut-misnet +application/vnd.fuzzysheet fzs +application/vnd.genomatix.tuxedo txd +application/vnd.gmx gmx +application/vnd.google-earth.kml+xml kml +application/vnd.google-earth.kmz kmz +application/vnd.grafeq gqf gqs +application/vnd.gridmp +application/vnd.groove-account gac +application/vnd.groove-help ghf +application/vnd.groove-identity-message gim +application/vnd.groove-injector grv +application/vnd.groove-tool-message gtm +application/vnd.groove-tool-template tpl +application/vnd.groove-vcard vcg +application/vnd.handheld-entertainment+xml zmm +application/vnd.hbci hbci +application/vnd.hcl-bireports +application/vnd.hhe.lesson-player les +application/vnd.hp-hpgl hpgl +application/vnd.hp-hpid hpid +application/vnd.hp-hps hps +application/vnd.hp-jlyt jlt +application/vnd.hp-pcl pcl +application/vnd.hp-pclxl pclxl +application/vnd.httphone +application/vnd.hydrostatix.sof-data sfd-hdstx +application/vnd.hzn-3d-crossword x3d +application/vnd.ibm.afplinedata +application/vnd.ibm.electronic-media +application/vnd.ibm.minipay mpy +application/vnd.ibm.modcap afp listafp list3820 +application/vnd.ibm.rights-management irm +application/vnd.ibm.secure-container sc +application/vnd.iccprofile icc icm +application/vnd.igloader igl +application/vnd.immervision-ivp ivp +application/vnd.immervision-ivu ivu +application/vnd.informedcontrol.rms+xml +application/vnd.intercon.formnet xpw xpx +application/vnd.intertrust.digibox +application/vnd.intertrust.nncp +application/vnd.intu.qbo qbo +application/vnd.intu.qfx qfx +application/vnd.iptc.g2.conceptitem+xml +application/vnd.iptc.g2.knowledgeitem+xml +application/vnd.iptc.g2.newsitem+xml +application/vnd.iptc.g2.packageitem+xml +application/vnd.ipunplugged.rcprofile rcprofile +application/vnd.irepository.package+xml irp +application/vnd.is-xpr xpr +application/vnd.jam jam +application/vnd.japannet-directory-service +application/vnd.japannet-jpnstore-wakeup +application/vnd.japannet-payment-wakeup +application/vnd.japannet-registration +application/vnd.japannet-registration-wakeup +application/vnd.japannet-setstore-wakeup +application/vnd.japannet-verification +application/vnd.japannet-verification-wakeup +application/vnd.jcp.javame.midlet-rms rms +application/vnd.jisp jisp +application/vnd.joost.joda-archive joda +application/vnd.kahootz ktz ktr +application/vnd.kde.karbon karbon +application/vnd.kde.kchart chrt +application/vnd.kde.kformula kfo +application/vnd.kde.kivio flw +application/vnd.kde.kontour kon +application/vnd.kde.kpresenter kpr kpt +application/vnd.kde.kspread ksp +application/vnd.kde.kword kwd kwt +application/vnd.kenameaapp htke +application/vnd.kidspiration kia +application/vnd.kinar kne knp +application/vnd.koan skp skd skt skm +application/vnd.kodak-descriptor sse +application/vnd.liberty-request+xml +application/vnd.llamagraphics.life-balance.desktop lbd +application/vnd.llamagraphics.life-balance.exchange+xml lbe +application/vnd.lotus-1-2-3 123 +application/vnd.lotus-approach apr +application/vnd.lotus-freelance pre +application/vnd.lotus-notes nsf +application/vnd.lotus-organizer org +application/vnd.lotus-screencam scm +application/vnd.lotus-wordpro lwp +application/vnd.macports.portpkg portpkg +application/vnd.marlin.drm.actiontoken+xml +application/vnd.marlin.drm.conftoken+xml +application/vnd.marlin.drm.license+xml +application/vnd.marlin.drm.mdcf +application/vnd.mcd mcd +application/vnd.medcalcdata mc1 +application/vnd.mediastation.cdkey cdkey +application/vnd.meridian-slingshot +application/vnd.mfer mwf +application/vnd.mfmp mfm +application/vnd.micrografx.flo flo +application/vnd.micrografx.igx igx +application/vnd.mif mif +application/vnd.minisoft-hp3000-save +application/vnd.mitsubishi.misty-guard.trustweb +application/vnd.mobius.daf daf +application/vnd.mobius.dis dis +application/vnd.mobius.mbk mbk +application/vnd.mobius.mqy mqy +application/vnd.mobius.msl msl +application/vnd.mobius.plc plc +application/vnd.mobius.txf txf +application/vnd.mophun.application mpn +application/vnd.mophun.certificate mpc +application/vnd.motorola.flexsuite +application/vnd.motorola.flexsuite.adsi +application/vnd.motorola.flexsuite.fis +application/vnd.motorola.flexsuite.gotap +application/vnd.motorola.flexsuite.kmr +application/vnd.motorola.flexsuite.ttc +application/vnd.motorola.flexsuite.wem +application/vnd.motorola.iprm +application/vnd.mozilla.xul+xml xul +application/vnd.ms-artgalry cil +application/vnd.ms-asf asf +application/vnd.ms-cab-compressed cab +application/vnd.ms-excel xls xlm xla xlc xlt xlw +application/vnd.ms-fontobject eot +application/vnd.ms-htmlhelp chm +application/vnd.ms-ims ims +application/vnd.ms-lrm lrm +application/vnd.ms-playready.initiator+xml +application/vnd.ms-powerpoint ppt pps pot +application/vnd.ms-project mpp mpt +application/vnd.ms-tnef +application/vnd.ms-wmdrm.lic-chlg-req +application/vnd.ms-wmdrm.lic-resp +application/vnd.ms-wmdrm.meter-chlg-req +application/vnd.ms-wmdrm.meter-resp +application/vnd.ms-works wps wks wcm wdb +application/vnd.ms-wpl wpl +application/vnd.ms-xpsdocument xps +application/vnd.mseq mseq +application/vnd.msign +application/vnd.multiad.creator +application/vnd.multiad.creator.cif +application/vnd.music-niff +application/vnd.musician mus +application/vnd.muvee.style msty +application/vnd.ncd.control +application/vnd.ncd.reference +application/vnd.nervana +application/vnd.netfpx +application/vnd.neurolanguage.nlu nlu +application/vnd.noblenet-directory nnd +application/vnd.noblenet-sealer nns +application/vnd.noblenet-web nnw +application/vnd.nokia.catalogs +application/vnd.nokia.conml+wbxml +application/vnd.nokia.conml+xml +application/vnd.nokia.isds-radio-presets +application/vnd.nokia.iptv.config+xml +application/vnd.nokia.landmark+wbxml +application/vnd.nokia.landmark+xml +application/vnd.nokia.landmarkcollection+xml +application/vnd.nokia.n-gage.ac+xml +application/vnd.nokia.n-gage.data ngdat +application/vnd.nokia.n-gage.symbian.install n-gage +application/vnd.nokia.ncd +application/vnd.nokia.pcd+wbxml +application/vnd.nokia.pcd+xml +application/vnd.nokia.radio-preset rpst +application/vnd.nokia.radio-presets rpss +application/vnd.novadigm.edm edm +application/vnd.novadigm.edx edx +application/vnd.novadigm.ext ext +application/vnd.oasis.opendocument.chart odc +application/vnd.oasis.opendocument.chart-template otc +application/vnd.oasis.opendocument.formula odf +application/vnd.oasis.opendocument.formula-template otf +application/vnd.oasis.opendocument.graphics odg +application/vnd.oasis.opendocument.graphics-template otg +application/vnd.oasis.opendocument.image odi +application/vnd.oasis.opendocument.image-template oti +application/vnd.oasis.opendocument.presentation odp +application/vnd.oasis.opendocument.presentation-template otp +application/vnd.oasis.opendocument.spreadsheet ods +application/vnd.oasis.opendocument.spreadsheet-template ots +application/vnd.oasis.opendocument.text odt +application/vnd.oasis.opendocument.text-master otm +application/vnd.oasis.opendocument.text-template ott +application/vnd.oasis.opendocument.text-web oth +application/vnd.obn +application/vnd.olpc-sugar xo +application/vnd.oma-scws-config +application/vnd.oma-scws-http-request +application/vnd.oma-scws-http-response +application/vnd.oma.bcast.associated-procedure-parameter+xml +application/vnd.oma.bcast.drm-trigger+xml +application/vnd.oma.bcast.imd+xml +application/vnd.oma.bcast.ltkm +application/vnd.oma.bcast.notification+xml +application/vnd.oma.bcast.provisioningtrigger +application/vnd.oma.bcast.sgboot +application/vnd.oma.bcast.sgdd+xml +application/vnd.oma.bcast.sgdu +application/vnd.oma.bcast.simple-symbol-container +application/vnd.oma.bcast.smartcard-trigger+xml +application/vnd.oma.bcast.sprov+xml +application/vnd.oma.bcast.stkm +application/vnd.oma.dcd +application/vnd.oma.dcdc +application/vnd.oma.dd2+xml dd2 +application/vnd.oma.drm.risd+xml +application/vnd.oma.group-usage-list+xml +application/vnd.oma.poc.detailed-progress-report+xml +application/vnd.oma.poc.final-report+xml +application/vnd.oma.poc.groups+xml +application/vnd.oma.poc.invocation-descriptor+xml +application/vnd.oma.poc.optimized-progress-report+xml +application/vnd.oma.xcap-directory+xml +application/vnd.omads-email+xml +application/vnd.omads-file+xml +application/vnd.omads-folder+xml +application/vnd.omaloc-supl-init +application/vnd.openofficeorg.extension oxt +application/vnd.osa.netdeploy +application/vnd.osgi.dp dp +application/vnd.otps.ct-kip+xml +application/vnd.palm prc pdb pqa oprc +application/vnd.paos.xml +application/vnd.pg.format str +application/vnd.pg.osasli ei6 +application/vnd.piaccess.application-licence +application/vnd.picsel efif +application/vnd.poc.group-advertisement+xml +application/vnd.pocketlearn plf +application/vnd.powerbuilder6 pbd +application/vnd.powerbuilder6-s +application/vnd.powerbuilder7 +application/vnd.powerbuilder7-s +application/vnd.powerbuilder75 +application/vnd.powerbuilder75-s +application/vnd.preminet +application/vnd.previewsystems.box box +application/vnd.proteus.magazine mgz +application/vnd.publishare-delta-tree qps +application/vnd.pvi.ptid1 ptid +application/vnd.pwg-multiplexed +application/vnd.pwg-xhtml-print+xml +application/vnd.qualcomm.brew-app-res +application/vnd.quark.quarkxpress qxd qxt qwd qwt qxl qxb +application/vnd.rapid +application/vnd.recordare.musicxml mxl +application/vnd.recordare.musicxml+xml +application/vnd.renlearn.rlprint +application/vnd.rn-realmedia rm +application/vnd.route66.link66+xml link66 +application/vnd.ruckus.download +application/vnd.s3sms +application/vnd.sbm.mid2 +application/vnd.scribus +application/vnd.sealed.3df +application/vnd.sealed.csf +application/vnd.sealed.doc +application/vnd.sealed.eml +application/vnd.sealed.mht +application/vnd.sealed.net +application/vnd.sealed.ppt +application/vnd.sealed.tiff +application/vnd.sealed.xls +application/vnd.sealedmedia.softseal.html +application/vnd.sealedmedia.softseal.pdf +application/vnd.seemail see +application/vnd.sema sema +application/vnd.semd semd +application/vnd.semf semf +application/vnd.shana.informed.formdata ifm +application/vnd.shana.informed.formtemplate itp +application/vnd.shana.informed.interchange iif +application/vnd.shana.informed.package ipk +application/vnd.simtech-mindmapper twd twds +application/vnd.smaf mmf +application/vnd.software602.filler.form+xml +application/vnd.software602.filler.form-xml-zip +application/vnd.solent.sdkm+xml sdkm sdkd +application/vnd.spotfire.dxp dxp +application/vnd.spotfire.sfs sfs +application/vnd.sss-cod +application/vnd.sss-dtf +application/vnd.sss-ntf +application/vnd.street-stream +application/vnd.sun.wadl+xml +application/vnd.sus-calendar sus susp +application/vnd.svd svd +application/vnd.swiftview-ics +application/vnd.syncml+xml xsm +application/vnd.syncml.dm+wbxml bdm +application/vnd.syncml.dm+xml xdm +application/vnd.syncml.ds.notification +application/vnd.tao.intent-module-archive tao +application/vnd.tmobile-livetv tmo +application/vnd.trid.tpt tpt +application/vnd.triscape.mxs mxs +application/vnd.trueapp tra +application/vnd.truedoc +application/vnd.ufdl ufd ufdl +application/vnd.uiq.theme utz +application/vnd.umajin umj +application/vnd.unity unityweb +application/vnd.uoml+xml uoml +application/vnd.uplanet.alert +application/vnd.uplanet.alert-wbxml +application/vnd.uplanet.bearer-choice +application/vnd.uplanet.bearer-choice-wbxml +application/vnd.uplanet.cacheop +application/vnd.uplanet.cacheop-wbxml +application/vnd.uplanet.channel +application/vnd.uplanet.channel-wbxml +application/vnd.uplanet.list +application/vnd.uplanet.list-wbxml +application/vnd.uplanet.listcmd +application/vnd.uplanet.listcmd-wbxml +application/vnd.uplanet.signal +application/vnd.vcx vcx +application/vnd.vd-study +application/vnd.vectorworks +application/vnd.vidsoft.vidconference +application/vnd.visio vsd vst vss vsw +application/vnd.visionary vis +application/vnd.vividence.scriptfile +application/vnd.vsf vsf +application/vnd.wap.sic +application/vnd.wap.slc +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo wtb +application/vnd.wfa.wsc +application/vnd.wmc +application/vnd.wmf.bootstrap +application/vnd.wordperfect wpd +application/vnd.wqd wqd +application/vnd.wrq-hp3000-labelled +application/vnd.wt.stf stf +application/vnd.wv.csp+wbxml +application/vnd.wv.csp+xml +application/vnd.wv.ssp+xml +application/vnd.xara xar +application/vnd.xfdl xfdl +application/vnd.xmi+xml +application/vnd.xmpie.cpkg +application/vnd.xmpie.dpkg +application/vnd.xmpie.plan +application/vnd.xmpie.ppkg +application/vnd.xmpie.xlim +application/vnd.yamaha.hv-dic hvd +application/vnd.yamaha.hv-script hvs +application/vnd.yamaha.hv-voice hvp +application/vnd.yamaha.smaf-audio saf +application/vnd.yamaha.smaf-phrase spf +application/vnd.yellowriver-custom-menu cmp +application/vnd.zzazz.deck+xml zaz +application/voicexml+xml vxml +application/watcherinfo+xml +application/whoispp-query +application/whoispp-response +application/winhlp hlp +application/wita +application/wordperfect5.1 +application/wsdl+xml wsdl +application/wspolicy+xml wspolicy +application/x-ace-compressed ace +application/x-bcpio bcpio +application/x-bittorrent torrent +application/x-bzip bz +application/x-bzip2 bz2 boz +application/x-cdlink vcd +application/x-chat chat +application/x-chess-pgn pgn +application/x-compress +application/x-cpio cpio +application/x-csh csh +application/x-director dcr dir dxr fgd +application/x-dvi dvi +application/x-futuresplash spl +application/x-gtar gtar +application/x-gzip +application/x-hdf hdf +application/x-latex latex +application/x-ms-wmd wmd +application/x-ms-wmz wmz +application/x-msaccess mdb +application/x-msbinder obd +application/x-mscardfile crd +application/x-msclip clp +application/x-msdownload exe dll com bat msi +application/x-msmediaview mvb m13 m14 +application/x-msmetafile wmf +application/x-msmoney mny +application/x-mspublisher pub +application/x-msschedule scd +application/x-msterminal trm +application/x-mswrite wri +application/x-netcdf nc cdf +application/x-pkcs12 p12 pfx +application/x-pkcs7-certificates p7b spc +application/x-pkcs7-certreqresp p7r +application/x-rar-compressed rar +application/x-sh sh +application/x-shar shar +application/x-shockwave-flash swf +application/x-stuffit sit +application/x-stuffitx sitx +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-texinfo texinfo texi +application/x-ustar ustar +application/x-wais-source src +application/x-x509-ca-cert der crt +application/x400-bp +application/xcap-att+xml +application/xcap-caps+xml +application/xcap-el+xml +application/xcap-error+xml +application/xcap-ns+xml +application/xenc+xml xenc +application/xhtml+xml xhtml xht +application/xml xml xsl +application/xml-dtd dtd +application/xml-external-parsed-entity +application/xmpp+xml +application/xop+xml xop +application/xslt+xml xslt +application/xspf+xml xspf +application/xv+xml mxml xhvml xvml xvm +application/zip zip +audio/32kadpcm +audio/3gpp +audio/3gpp2 +audio/ac3 +audio/amr +audio/amr-wb +audio/amr-wb+ +audio/asc +audio/basic au snd +audio/bv16 +audio/bv32 +audio/clearmode +audio/cn +audio/dat12 +audio/dls +audio/dsr-es201108 +audio/dsr-es202050 +audio/dsr-es202211 +audio/dsr-es202212 +audio/dvi4 +audio/eac3 +audio/evrc +audio/evrc-qcp +audio/evrc0 +audio/evrc1 +audio/evrcb +audio/evrcb0 +audio/evrcb1 +audio/evrcwb +audio/evrcwb0 +audio/evrcwb1 +audio/g722 +audio/g7221 +audio/g723 +audio/g726-16 +audio/g726-24 +audio/g726-32 +audio/g726-40 +audio/g728 +audio/g729 +audio/g7291 +audio/g729d +audio/g729e +audio/gsm +audio/gsm-efr +audio/ilbc +audio/l16 +audio/l20 +audio/l24 +audio/l8 +audio/lpc +audio/midi mid midi kar rmi +audio/mobile-xmf +audio/mp4 mp4a +audio/mp4a-latm +audio/mpa +audio/mpa-robust +audio/mpeg mpga mp2 mp2a mp3 m2a m3a +audio/mpeg4-generic +audio/ogg oga ogg spx +audio/parityfec +audio/pcma +audio/pcmu +audio/prs.sid +audio/qcelp +audio/red +audio/rtp-enc-aescm128 +audio/rtp-midi +audio/rtx +audio/smv +audio/smv0 +audio/smv-qcp +audio/sp-midi +audio/t140c +audio/t38 +audio/telephone-event +audio/tone +audio/ulpfec +audio/vdvi +audio/vmr-wb +audio/vnd.3gpp.iufp +audio/vnd.4sb +audio/vnd.audiokoz +audio/vnd.celp +audio/vnd.cisco.nse +audio/vnd.cmles.radio-events +audio/vnd.cns.anp1 +audio/vnd.cns.inf1 +audio/vnd.digital-winds eol +audio/vnd.dlna.adts +audio/vnd.dolby.mlp +audio/vnd.dts dts +audio/vnd.dts.hd dtshd +audio/vnd.everad.plj +audio/vnd.hns.audio +audio/vnd.lucent.voice lvp +audio/vnd.ms-playready.media.pya pya +audio/vnd.nokia.mobile-xmf +audio/vnd.nortel.vbk +audio/vnd.nuera.ecelp4800 ecelp4800 +audio/vnd.nuera.ecelp7470 ecelp7470 +audio/vnd.nuera.ecelp9600 ecelp9600 +audio/vnd.octel.sbc +audio/vnd.qcelp +audio/vnd.rhetorex.32kadpcm +audio/vnd.sealedmedia.softseal.mpeg +audio/vnd.vmx.cvsd +audio/vorbis +audio/vorbis-config +audio/wav wav +audio/x-aiff aif aiff aifc +audio/x-mpegurl m3u +audio/x-ms-wax wax +audio/x-ms-wma wma +audio/x-pn-realaudio ram ra +audio/x-pn-realaudio-plugin rmp +audio/x-wav wav +chemical/x-cdx cdx +chemical/x-cif cif +chemical/x-cmdf cmdf +chemical/x-cml cml +chemical/x-csml csml +chemical/x-pdb pdb +chemical/x-xyz xyz +image/bmp bmp +image/cgm cgm +image/fits +image/g3fax g3 +image/gif gif +image/ief ief +image/jp2 +image/jpeg jpeg jpg jpe +image/jpm +image/jpx +image/naplps +image/png png +image/prs.btif btif +image/prs.pti +image/svg+xml svg svgz +image/t38 +image/tiff tiff tif +image/tiff-fx +image/vnd.adobe.photoshop psd +image/vnd.cns.inf2 +image/vnd.djvu djvu djv +image/vnd.dwg dwg +image/vnd.dxf dxf +image/vnd.fastbidsheet fbs +image/vnd.fpx fpx +image/vnd.fst fst +image/vnd.fujixerox.edmics-mmr mmr +image/vnd.fujixerox.edmics-rlc rlc +image/vnd.globalgraphics.pgb +image/vnd.microsoft.icon +image/vnd.mix +image/vnd.ms-modi mdi +image/vnd.net-fpx npx +image/vnd.sealed.png +image/vnd.sealedmedia.softseal.gif +image/vnd.sealedmedia.softseal.jpg +image/vnd.svf +image/vnd.wap.wbmp wbmp +image/vnd.xiff xif +image/x-cmu-raster ras +image/x-cmx cmx +image/x-icon ico +image/x-pcx pcx +image/x-pict pic pct +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +message/cpim +message/delivery-status +message/disposition-notification +message/external-body +message/global +message/global-delivery-status +message/global-disposition-notification +message/global-headers +message/http +message/news +message/partial +message/rfc822 eml mime +message/s-http +message/sip +message/sipfrag +message/tracking-status +message/vnd.si.simp +model/iges igs iges +model/mesh msh mesh silo +model/vnd.dwf dwf +model/vnd.flatland.3dml +model/vnd.gdl gdl +model/vnd.gs.gdl +model/vnd.gtw gtw +model/vnd.moml+xml +model/vnd.mts mts +model/vnd.parasolid.transmit.binary +model/vnd.parasolid.transmit.text +model/vnd.vtu vtu +model/vrml wrl vrml +multipart/alternative +multipart/appledouble +multipart/byteranges +multipart/digest +multipart/encrypted +multipart/form-data +multipart/header-set +multipart/mixed +multipart/parallel +multipart/related +multipart/report +multipart/signed +multipart/voice-message +text/calendar ics ifb +text/css css +text/csv csv +text/directory +text/dns +text/enriched +text/html html htm +text/parityfec +text/plain txt text conf def list log in +text/prs.fallenstein.rst +text/prs.lines.tag dsc +text/red +text/rfc822-headers +text/richtext rtx +text/rtf +text/rtp-enc-aescm128 +text/rtx +text/sgml sgml sgm +text/t140 +text/tab-separated-values tsv +text/troff t tr roff man me ms +text/ulpfec +text/uri-list uri uris urls +text/vnd.abc +text/vnd.curl +text/vnd.dmclientscript +text/vnd.esmertec.theme-descriptor +text/vnd.fly fly +text/vnd.fmi.flexstor flx +text/vnd.graphviz gv +text/vnd.in3d.3dml 3dml +text/vnd.in3d.spot spot +text/vnd.iptc.newsml +text/vnd.iptc.nitf +text/vnd.latex-z +text/vnd.motorola.reflex +text/vnd.ms-mediapackage +text/vnd.net2phone.commcenter.command +text/vnd.si.uricatalogue +text/vnd.sun.j2me.app-descriptor jad +text/vnd.trolltech.linguist +text/vnd.wap.si +text/vnd.wap.sl +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/x-asm s asm +text/x-c c cc cxx cpp h hh dic +text/x-fortran f for f77 f90 +text/x-pascal p pas +text/x-java-source java +text/x-setext etx +text/x-uuencode uu +text/x-vcalendar vcs +text/x-vcard vcf +text/xml +text/xml-external-parsed-entity +video/3gpp 3gp +video/3gpp-tt +video/3gpp2 3g2 +video/bmpeg +video/bt656 +video/celb +video/dv +video/h261 h261 +video/h263 h263 +video/h263-1998 +video/h263-2000 +video/h264 h264 +video/jpeg jpgv +video/jpeg2000 +video/jpm jpm jpgm +video/mj2 mj2 mjp2 +video/mp1s +video/mp2p +video/mp2t +video/mp4 mp4 mp4v mpg4 +video/mp4v-es +video/mpeg mpeg mpg mpe m1v m2v +video/mpeg4-generic +video/mpv +video/nv +video/ogg ogv +video/parityfec +video/pointer +video/quicktime qt mov +video/raw +video/rtp-enc-aescm128 +video/rtx +video/smpte292m +video/ulpfec +video/vc1 +video/vnd.cctv +video/vnd.dlna.mpeg-tts +video/vnd.fvt fvt +video/vnd.hns.video +video/vnd.iptvforum.1dparityfec-1010 +video/vnd.iptvforum.1dparityfec-2005 +video/vnd.iptvforum.2dparityfec-1010 +video/vnd.iptvforum.2dparityfec-2005 +video/vnd.iptvforum.ttsavc +video/vnd.iptvforum.ttsmpeg2 +video/vnd.motorola.video +video/vnd.motorola.videop +video/vnd.mpegurl mxu m4u +video/vnd.ms-playready.media.pyv pyv +video/vnd.nokia.interleaved-multimedia +video/vnd.nokia.videovoip +video/vnd.objectvideo +video/vnd.sealed.mpeg1 +video/vnd.sealed.mpeg4 +video/vnd.sealed.swf +video/vnd.sealedmedia.softseal.mov +video/vnd.vivo viv +video/x-fli fli +video/x-ms-asf asf asx +video/x-ms-wm wm +video/x-ms-wmv wmv +video/x-ms-wmx wmx +video/x-ms-wvx wvx +video/x-msvideo avi +video/x-sgi-movie movie +x-conference/x-cooltalk ice diff --git a/conf/storage.conf b/conf/storage.conf new file mode 100644 index 0000000..cd1f707 --- /dev/null +++ b/conf/storage.conf @@ -0,0 +1,273 @@ +# is this config file disabled +# false for enabled +# true for disabled +disabled=false + +# the name of the group this storage server belongs to +group_name=group1 + +# bind an address of this host +# empty for bind all addresses of this host +bind_addr= + +# if bind an address of this host when connect to other servers +# (this storage server as a client) +# true for binding the address configed by above parameter: "bind_addr" +# false for binding any address of this host +client_bind=true + +# the storage server port +port=23000 + +# connect timeout in seconds +# default value is 30s +connect_timeout=30 + +# network timeout in seconds +# default value is 30s +network_timeout=60 + +# heart beat interval in seconds +heart_beat_interval=30 + +# disk usage report interval in seconds +stat_report_interval=60 + +# the base path to store data and log files +base_path=/home/yuqing/fastdfs + +# max concurrent connections the server supported +# default value is 256 +# more max_connections means more memory will be used +max_connections=256 + +# the buff size to recv / send data +# this parameter must more than 8KB +# default value is 64KB +# since V2.00 +buff_size = 256KB + +# accept thread count +# default value is 1 +# since V4.07 +accept_threads=1 + +# work thread count, should <= max_connections +# work thread deal network io +# default value is 4 +# since V2.00 +work_threads=4 + +# if disk read / write separated +## false for mixed read and write +## true for separated read and write +# default value is true +# since V2.00 +disk_rw_separated = true + +# disk reader thread count per store base path +# for mixed read / write, this parameter can be 0 +# default value is 1 +# since V2.00 +disk_reader_threads = 1 + +# disk writer thread count per store base path +# for mixed read / write, this parameter can be 0 +# default value is 1 +# since V2.00 +disk_writer_threads = 1 + +# when no entry to sync, try read binlog again after X milliseconds +# must > 0, default value is 200ms +sync_wait_msec=50 + +# after sync a file, usleep milliseconds +# 0 for sync successively (never call usleep) +sync_interval=0 + +# storage sync start time of a day, time format: Hour:Minute +# Hour from 0 to 23, Minute from 0 to 59 +sync_start_time=00:00 + +# storage sync end time of a day, time format: Hour:Minute +# Hour from 0 to 23, Minute from 0 to 59 +sync_end_time=23:59 + +# write to the mark file after sync N files +# default value is 500 +write_mark_file_freq=500 + +# path(disk or mount point) count, default value is 1 +store_path_count=1 + +# store_path#, based 0, if store_path0 not exists, it's value is base_path +# the paths must be exist +store_path0=/home/yuqing/fastdfs +#store_path1=/home/yuqing/fastdfs2 + +# subdir_count * subdir_count directories will be auto created under each +# store_path (disk), value can be 1 to 256, default value is 256 +subdir_count_per_path=256 + +# tracker_server can ocur more than once, and tracker_server format is +# "host:port", host can be hostname or ip address +tracker_server=192.168.209.121:22122 + +#standard log level as syslog, case insensitive, value list: +### emerg for emergency +### alert +### crit for critical +### error +### warn for warning +### notice +### info +### debug +log_level=info + +#unix group name to run this program, +#not set (empty) means run by the group of current user +run_by_group= + +#unix username to run this program, +#not set (empty) means run by current user +run_by_user= + +# allow_hosts can ocur more than once, host can be hostname or ip address, +# "*" means match all ip addresses, can use range like this: 10.0.1.[1-15,20] or +# host[01-08,20-25].domain.com, for example: +# allow_hosts=10.0.1.[1-15,20] +# allow_hosts=host[01-08,20-25].domain.com +allow_hosts=* + +# the mode of the files distributed to the data path +# 0: round robin(default) +# 1: random, distributted by hash code +file_distribute_path_mode=0 + +# valid when file_distribute_to_path is set to 0 (round robin), +# when the written file count reaches this number, then rotate to next path +# default value is 100 +file_distribute_rotate_count=100 + +# call fsync to disk when write big file +# 0: never call fsync +# other: call fsync when written bytes >= this bytes +# default value is 0 (never call fsync) +fsync_after_written_bytes=0 + +# sync log buff to disk every interval seconds +# must > 0, default value is 10 seconds +sync_log_buff_interval=10 + +# sync binlog buff / cache to disk every interval seconds +# default value is 60 seconds +sync_binlog_buff_interval=10 + +# sync storage stat info to disk every interval seconds +# default value is 300 seconds +sync_stat_file_interval=300 + +# thread stack size, should >= 512KB +# default value is 512KB +thread_stack_size=512KB + +# the priority as a source server for uploading file. +# the lower this value, the higher its uploading priority. +# default value is 10 +upload_priority=10 + +# the NIC alias prefix, such as eth in Linux, you can see it by ifconfig -a +# multi aliases split by comma. empty value means auto set by OS type +# default values is empty +if_alias_prefix= + +# if check file duplicate, when set to true, use FastDHT to store file indexes +# 1 or yes: need check +# 0 or no: do not check +# default value is 0 +check_file_duplicate=0 + +# file signature method for check file duplicate +## hash: four 32 bits hash code +## md5: MD5 signature +# default value is hash +# since V4.01 +file_signature_method=hash + +# namespace for storing file indexes (key-value pairs) +# this item must be set when check_file_duplicate is true / on +key_namespace=FastDFS + +# set keep_alive to 1 to enable persistent connection with FastDHT servers +# default value is 0 (short connection) +keep_alive=0 + +# you can use "#include filename" (not include double quotes) directive to +# load FastDHT server list, when the filename is a relative path such as +# pure filename, the base path is the base path of current/this config file. +# must set FastDHT server list when check_file_duplicate is true / on +# please see INSTALL of FastDHT for detail +##include /home/yuqing/fastdht/conf/fdht_servers.conf + +# if log to access log +# default value is false +# since V4.00 +use_access_log = false + +# if rotate the access log every day +# default value is false +# since V4.00 +rotate_access_log = false + +# rotate access log time base, time format: Hour:Minute +# Hour from 0 to 23, Minute from 0 to 59 +# default value is 00:00 +# since V4.00 +access_log_rotate_time=00:00 + +# if rotate the error log every day +# default value is false +# since V4.02 +rotate_error_log = false + +# rotate error log time base, time format: Hour:Minute +# Hour from 0 to 23, Minute from 0 to 59 +# default value is 00:00 +# since V4.02 +error_log_rotate_time=00:00 + +# rotate access log when the log file exceeds this size +# 0 means never rotates log file by log file size +# default value is 0 +# since V4.02 +rotate_access_log_size = 0 + +# rotate error log when the log file exceeds this size +# 0 means never rotates log file by log file size +# default value is 0 +# since V4.02 +rotate_error_log_size = 0 + +# if skip the invalid record when sync file +# default value is false +# since V4.02 +file_sync_skip_invalid_record=false + +# if use connection pool +# default value is false +# since V4.05 +use_connection_pool = false + +# connections whose the idle time exceeds this time will be closed +# unit: second +# default value is 3600 +# since V4.05 +connection_pool_max_idle_time = 3600 + +# use the ip address of this storage server if domain_name is empty, +# else this domain name will ocur in the url redirected by the tracker server +http.domain_name= + +# the port of the web server on this storage server +http.server_port=8888 + diff --git a/conf/storage_ids.conf b/conf/storage_ids.conf new file mode 100644 index 0000000..7206056 --- /dev/null +++ b/conf/storage_ids.conf @@ -0,0 +1,3 @@ +# +# 100001 group1 192.168.0.196 +# 100002 group1 192.168.0.116 diff --git a/conf/tracker.conf b/conf/tracker.conf new file mode 100644 index 0000000..c7fe941 --- /dev/null +++ b/conf/tracker.conf @@ -0,0 +1,260 @@ +# is this config file disabled +# false for enabled +# true for disabled +disabled=false + +# bind an address of this host +# empty for bind all addresses of this host +bind_addr= + +# the tracker server port +port=22122 + +# connect timeout in seconds +# default value is 30s +connect_timeout=30 + +# network timeout in seconds +# default value is 30s +network_timeout=60 + +# the base path to store data and log files +base_path=/home/yuqing/fastdfs + +# max concurrent connections this server supported +max_connections=256 + +# accept thread count +# default value is 1 +# since V4.07 +accept_threads=1 + +# work thread count, should <= max_connections +# default value is 4 +# since V2.00 +work_threads=4 + +# the method of selecting group to upload files +# 0: round robin +# 1: specify group +# 2: load balance, select the max free space group to upload file +store_lookup=2 + +# which group to upload file +# when store_lookup set to 1, must set store_group to the group name +store_group=group2 + +# which storage server to upload file +# 0: round robin (default) +# 1: the first server order by ip address +# 2: the first server order by priority (the minimal) +store_server=0 + +# which path(means disk or mount point) of the storage server to upload file +# 0: round robin +# 2: load balance, select the max free space path to upload file +store_path=0 + +# which storage server to download file +# 0: round robin (default) +# 1: the source storage server which the current file uploaded to +download_server=0 + +# reserved storage space for system or other applications. +# if the free(available) space of any stoarge server in +# a group <= reserved_storage_space, +# no file can be uploaded to this group. +# bytes unit can be one of follows: +### G or g for gigabyte(GB) +### M or m for megabyte(MB) +### K or k for kilobyte(KB) +### no unit for byte(B) +### XX.XX% as ratio such as reserved_storage_space = 10% +reserved_storage_space = 10% + +#standard log level as syslog, case insensitive, value list: +### emerg for emergency +### alert +### crit for critical +### error +### warn for warning +### notice +### info +### debug +log_level=info + +#unix group name to run this program, +#not set (empty) means run by the group of current user +run_by_group= + +#unix username to run this program, +#not set (empty) means run by current user +run_by_user= + +# allow_hosts can ocur more than once, host can be hostname or ip address, +# "*" means match all ip addresses, can use range like this: 10.0.1.[1-15,20] or +# host[01-08,20-25].domain.com, for example: +# allow_hosts=10.0.1.[1-15,20] +# allow_hosts=host[01-08,20-25].domain.com +allow_hosts=* + +# sync log buff to disk every interval seconds +# default value is 10 seconds +sync_log_buff_interval = 10 + +# check storage server alive interval seconds +check_active_interval = 120 + +# thread stack size, should >= 64KB +# default value is 64KB +thread_stack_size = 64KB + +# auto adjust when the ip address of the storage server changed +# default value is true +storage_ip_changed_auto_adjust = true + +# storage sync file max delay seconds +# default value is 86400 seconds (one day) +# since V2.00 +storage_sync_file_max_delay = 86400 + +# the max time of storage sync a file +# default value is 300 seconds +# since V2.00 +storage_sync_file_max_time = 300 + +# if use a trunk file to store several small files +# default value is false +# since V3.00 +use_trunk_file = false + +# the min slot size, should <= 4KB +# default value is 256 bytes +# since V3.00 +slot_min_size = 256 + +# the max slot size, should > slot_min_size +# store the upload file to trunk file when it's size <= this value +# default value is 16MB +# since V3.00 +slot_max_size = 16MB + +# the trunk file size, should >= 4MB +# default value is 64MB +# since V3.00 +trunk_file_size = 64MB + +# if create trunk file advancely +# default value is false +# since V3.06 +trunk_create_file_advance = false + +# the time base to create trunk file +# the time format: HH:MM +# default value is 02:00 +# since V3.06 +trunk_create_file_time_base = 02:00 + +# the interval of create trunk file, unit: second +# default value is 38400 (one day) +# since V3.06 +trunk_create_file_interval = 86400 + +# the threshold to create trunk file +# when the free trunk file size less than the threshold, will create +# the trunk files +# default value is 0 +# since V3.06 +trunk_create_file_space_threshold = 20G + +# if check trunk space occupying when loading trunk free spaces +# the occupied spaces will be ignored +# default value is false +# since V3.09 +# NOTICE: set this parameter to true will slow the loading of trunk spaces +# when startup. you should set this parameter to true when neccessary. +trunk_init_check_occupying = false + +# if ignore storage_trunk.dat, reload from trunk binlog +# default value is false +# since V3.10 +# set to true once for version upgrade when your version less than V3.10 +trunk_init_reload_from_binlog = false + +# the min interval for compressing the trunk binlog file +# unit: second +# default value is 0, 0 means never compress +# FastDFS compress the trunk binlog when trunk init and trunk destroy +# recommand to set this parameter to 86400 (one day) +# since V5.01 +trunk_compress_binlog_min_interval = 0 + +# if use storage ID instead of IP address +# default value is false +# since V4.00 +use_storage_id = false + +# specify storage ids filename, can use relative or absolute path +# since V4.00 +storage_ids_filename = storage_ids.conf + +# id type of the storage server in the filename, values are: +## ip: the ip address of the storage server +## id: the server id of the storage server +# this paramter is valid only when use_storage_id set to true +# default value is ip +# since V4.03 +id_type_in_filename = ip + +# if store slave file use symbol link +# default value is false +# since V4.01 +store_slave_file_use_link = false + +# if rotate the error log every day +# default value is false +# since V4.02 +rotate_error_log = false + +# rotate error log time base, time format: Hour:Minute +# Hour from 0 to 23, Minute from 0 to 59 +# default value is 00:00 +# since V4.02 +error_log_rotate_time=00:00 + +# rotate error log when the log file exceeds this size +# 0 means never rotates log file by log file size +# default value is 0 +# since V4.02 +rotate_error_log_size = 0 + +# if use connection pool +# default value is false +# since V4.05 +use_connection_pool = false + +# connections whose the idle time exceeds this time will be closed +# unit: second +# default value is 3600 +# since V4.05 +connection_pool_max_idle_time = 3600 + +# HTTP port on this tracker server +http.server_port=8080 + +# check storage HTTP server alive interval seconds +# <= 0 for never check +# default value is 30 +http.check_alive_interval=30 + +# check storage HTTP server alive type, values are: +# tcp : connect to the storge server with HTTP port only, +# do not request and get response +# http: storage check alive url must return http status 200 +# default value is tcp +http.check_alive_type=tcp + +# check storage HTTP server alive uri/url +# NOTE: storage embed HTTP server support uri: /status.html +http.check_alive_uri=/status.html + diff --git a/init.d/fdfs_storaged b/init.d/fdfs_storaged new file mode 100755 index 0000000..46e55ee --- /dev/null +++ b/init.d/fdfs_storaged @@ -0,0 +1,82 @@ +#!/bin/bash +# +# fdfs_storaged Starts fdfs_storaged +# +# +# chkconfig: 2345 99 01 +# description: FastDFS storage server +### BEGIN INIT INFO +# Provides: $fdfs_storaged +### END INIT INFO + +# Source function library. +. /etc/init.d/functions + +PRG=/usr/local/bin/fdfs_storaged +CONF=/etc/fdfs/storage.conf + +if [ ! -f $PRG ]; then + echo "file $PRG does not exist!" + exit 2 +fi + +if [ ! -f /usr/local/bin/stop.sh ]; then + echo "file /usr/local/bin/stop.sh does not exist!" + exit 2 +fi + +if [ ! -f /usr/local/bin/restart.sh ]; then + echo "file /usr/local/bin/restart.sh does not exist!" + exit 2 +fi + +if [ ! -f $CONF ]; then + echo "file $CONF does not exist!" + exit 2 +fi + +CMD="$PRG $CONF" +RETVAL=0 + +start() { + echo -n $"Starting FastDFS storage server: " + $CMD & + RETVAL=$? + echo + return $RETVAL +} +stop() { + /usr/local/bin/stop.sh $CMD + RETVAL=$? + return $RETVAL +} +rhstatus() { + status fdfs_storaged +} +restart() { + /usr/local/bin/restart.sh $CMD & +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + rhstatus + ;; + restart|reload) + restart + ;; + condrestart) + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $? + diff --git a/init.d/fdfs_trackerd b/init.d/fdfs_trackerd new file mode 100755 index 0000000..f9cd21d --- /dev/null +++ b/init.d/fdfs_trackerd @@ -0,0 +1,82 @@ +#!/bin/bash +# +# fdfs_trackerd Starts fdfs_trackerd +# +# +# chkconfig: 2345 99 01 +# description: FastDFS tracker server +### BEGIN INIT INFO +# Provides: $fdfs_trackerd +### END INIT INFO + +# Source function library. +. /etc/init.d/functions + +PRG=/usr/local/bin/fdfs_trackerd +CONF=/etc/fdfs/tracker.conf + +if [ ! -f $PRG ]; then + echo "file $PRG does not exist!" + exit 2 +fi + +if [ ! -f /usr/local/bin/stop.sh ]; then + echo "file /usr/local/bin/stop.sh does not exist!" + exit 2 +fi + +if [ ! -f /usr/local/bin/restart.sh ]; then + echo "file /usr/local/bin/restart.sh does not exist!" + exit 2 +fi + +if [ ! -f $CONF ]; then + echo "file $CONF does not exist!" + exit 2 +fi + +CMD="$PRG $CONF" +RETVAL=0 + +start() { + echo -n $"Starting FastDFS tracker server: " + $CMD & + RETVAL=$? + echo + return $RETVAL +} +stop() { + /usr/local/bin/stop.sh $CMD + RETVAL=$? + return $RETVAL +} +rhstatus() { + status fdfs_trackerd +} +restart() { + /usr/local/bin/restart.sh $CMD & +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + rhstatus + ;; + restart|reload) + restart + ;; + condrestart) + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $? + diff --git a/make.sh b/make.sh new file mode 100755 index 0000000..6b6f11a --- /dev/null +++ b/make.sh @@ -0,0 +1,208 @@ +tmp_src_filename=fdfs_check_bits.c +cat < $tmp_src_filename +#include +#include +#include +int main() +{ + printf("%d\n", (int)sizeof(long)); + printf("%d\n", (int)sizeof(off_t)); + return 0; +} +EOF + +gcc -D_FILE_OFFSET_BITS=64 -o a.out $tmp_src_filename +output=$(./a.out) + +if [ -f /bin/expr ]; then + EXPR=/bin/expr +else + EXPR=/usr/bin/expr +fi + +count=0 +int_bytes=4 +off_bytes=8 +for col in $output; do + if [ $count -eq 0 ]; then + int_bytes=$col + else + off_bytes=$col + fi + + count=$($EXPR $count + 1) +done + +/bin/rm -f a.out $tmp_src_filename +if [ "$int_bytes" -eq 8 ]; then + OS_BITS=64 +else + OS_BITS=32 +fi + +if [ "$off_bytes" -eq 8 ]; then + OFF_BITS=64 +else + OFF_BITS=32 +fi + +cat < common/_os_bits.h +#ifndef _OS_BITS_H +#define _OS_BITS_H + +#define OS_BITS $OS_BITS +#define OFF_BITS $OFF_BITS + +#endif +EOF + +ENABLE_STATIC_LIB=0 +ENABLE_SHARED_LIB=1 +TARGET_PREFIX=/usr/local +TARGET_CONF_PATH=/etc/fdfs + +#WITH_LINUX_SERVICE=1 + +DEBUG_FLAG=1 + +CFLAGS='-Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE' +if [ "$DEBUG_FLAG" = "1" ]; then + CFLAGS="$CFLAGS -g -O -DDEBUG_FLAG" +else + CFLAGS="$CFLAGS -O3" +fi + +LIBS='' +uname=$(uname) +if [ "$uname" = "Linux" ]; then + CFLAGS="$CFLAGS -DOS_LINUX -DIOEVENT_USE_EPOLL" +elif [ "$uname" = "FreeBSD" ]; then + CFLAGS="$CFLAGS -DOS_FREEBSD -DIOEVENT_USE_KQUEUE" +elif [ "$uname" = "SunOS" ]; then + CFLAGS="$CFLAGS -DOS_SUNOS -D_THREAD_SAFE -DIOEVENT_USE_PORT" + LIBS="$LIBS -lsocket -lnsl -lresolv" + export CC=gcc +elif [ "$uname" = "AIX" ]; then + CFLAGS="$CFLAGS -DOS_AIX -D_THREAD_SAFE" + export CC=gcc +elif [ "$uname" = "HP-UX" ]; then + CFLAGS="$CFLAGS -DOS_HPUX" +fi + +have_pthread=0 +if [ -f /usr/lib/libpthread.so ] || [ -f /usr/local/lib/libpthread.so ] || [ -f /lib64/libpthread.so ] || [ -f /usr/lib64/libpthread.so ] || [ -f /usr/lib/libpthread.a ] || [ -f /usr/local/lib/libpthread.a ] || [ -f /lib64/libpthread.a ] || [ -f /usr/lib64/libpthread.a ]; then + LIBS="$LIBS -lpthread" + have_pthread=1 +elif [ "$uname" = "HP-UX" ]; then + lib_path="/usr/lib/hpux$OS_BITS" + if [ -f $lib_path/libpthread.so ]; then + LIBS="-L$lib_path -lpthread" + have_pthread=1 + fi +elif [ "$uname" = "FreeBSD" ]; then + if [ -f /usr/lib/libc_r.so ]; then + line=$(nm -D /usr/lib/libc_r.so | grep pthread_create | grep -w T) + if [ $? -eq 0 ]; then + LIBS="$LIBS -lc_r" + have_pthread=1 + fi + elif [ -f /lib64/libc_r.so ]; then + line=$(nm -D /lib64/libc_r.so | grep pthread_create | grep -w T) + if [ $? -eq 0 ]; then + LIBS="$LIBS -lc_r" + have_pthread=1 + fi + elif [ -f /usr/lib64/libc_r.so ]; then + line=$(nm -D /usr/lib64/libc_r.so | grep pthread_create | grep -w T) + if [ $? -eq 0 ]; then + LIBS="$LIBS -lc_r" + have_pthread=1 + fi + fi +fi + +if [ $have_pthread -eq 0 ]; then + /sbin/ldconfig -p | fgrep libpthread.so > /dev/null + if [ $? -eq 0 ]; then + LIBS="$LIBS -lpthread" + else + echo -E 'Require pthread lib, please check!' + exit 2 + fi +fi + +TRACKER_EXTRA_OBJS='' +STORAGE_EXTRA_OBJS='' +if [ "$DEBUG_FLAG" = "1" ]; then + TRACKER_EXTRA_OBJS="$TRACKER_EXTRA_OBJS tracker_dump.o" + STORAGE_EXTRA_OBJS="$STORAGE_EXTRA_OBJS storage_dump.o" + + if [ "$uname" = "Linux" ]; then + LIBS="$LIBS -ldl -rdynamic" + TRACKER_EXTRA_OBJS="$TRACKER_EXTRA_OBJS ../common/linux_stack_trace.o" + STORAGE_EXTRA_OBJS="$STORAGE_EXTRA_OBJS ../common/linux_stack_trace.o" + fi +fi + +cd tracker +cp Makefile.in Makefile +perl -pi -e "s#\\\$\(CFLAGS\)#$CFLAGS#g" Makefile +perl -pi -e "s#\\\$\(LIBS\)#$LIBS#g" Makefile +perl -pi -e "s#\\\$\(TARGET_PREFIX\)#$TARGET_PREFIX#g" Makefile +perl -pi -e "s#\\\$\(TRACKER_EXTRA_OBJS\)#$TRACKER_EXTRA_OBJS#g" Makefile +perl -pi -e "s#\\\$\(TARGET_CONF_PATH\)#$TARGET_CONF_PATH#g" Makefile +make $1 $2 + +cd ../storage +cp Makefile.in Makefile +perl -pi -e "s#\\\$\(CFLAGS\)#$CFLAGS#g" Makefile +perl -pi -e "s#\\\$\(LIBS\)#$LIBS#g" Makefile +perl -pi -e "s#\\\$\(TARGET_PREFIX\)#$TARGET_PREFIX#g" Makefile +perl -pi -e "s#\\\$\(STORAGE_EXTRA_OBJS\)#$STORAGE_EXTRA_OBJS#g" Makefile +perl -pi -e "s#\\\$\(TARGET_CONF_PATH\)#$TARGET_CONF_PATH#g" Makefile +make $1 $2 + +cd ../client +cp Makefile.in Makefile +perl -pi -e "s#\\\$\(CFLAGS\)#$CFLAGS#g" Makefile +perl -pi -e "s#\\\$\(LIBS\)#$LIBS#g" Makefile +perl -pi -e "s#\\\$\(TARGET_PREFIX\)#$TARGET_PREFIX#g" Makefile +perl -pi -e "s#\\\$\(TARGET_CONF_PATH\)#$TARGET_CONF_PATH#g" Makefile +perl -pi -e "s#\\\$\(ENABLE_STATIC_LIB\)#$ENABLE_STATIC_LIB#g" Makefile +perl -pi -e "s#\\\$\(ENABLE_SHARED_LIB\)#$ENABLE_SHARED_LIB#g" Makefile + +cp fdfs_link_library.sh.in fdfs_link_library.sh +perl -pi -e "s#\\\$\(TARGET_PREFIX\)#$TARGET_PREFIX#g" fdfs_link_library.sh +make $1 $2 + +cd test +cp Makefile.in Makefile +perl -pi -e "s#\\\$\(CFLAGS\)#$CFLAGS#g" Makefile +perl -pi -e "s#\\\$\(LIBS\)#$LIBS#g" Makefile +perl -pi -e "s#\\\$\(TARGET_PREFIX\)#$TARGET_PREFIX#g" Makefile +cd .. + +if [ "$1" = "install" ]; then + cd .. + cp -f restart.sh $TARGET_PREFIX/bin + cp -f stop.sh $TARGET_PREFIX/bin + + if [ "$uname" = "Linux" ]; then + if [ "$WITH_LINUX_SERVICE" = "1" ]; then + if [ ! -d /etc/fdfs ]; then + mkdir -p /etc/fdfs + cp -f conf/tracker.conf /etc/fdfs/ + cp -f conf/storage.conf /etc/fdfs/ + cp -f conf/client.conf /etc/fdfs/ + cp -f conf/http.conf /etc/fdfs/ + cp -f conf/mime.types /etc/fdfs/ + fi + + cp -f init.d/fdfs_trackerd /etc/rc.d/init.d/ + cp -f init.d/fdfs_storaged /etc/rc.d/init.d/ + /sbin/chkconfig --add fdfs_trackerd + /sbin/chkconfig --add fdfs_storaged + fi + fi +fi + diff --git a/php_client/README b/php_client/README new file mode 100644 index 0000000..6ef7855 --- /dev/null +++ b/php_client/README @@ -0,0 +1,2107 @@ + +Copyright (C) 2008 Happy Fish / YuQing + +FastDFS client php extension may be copied only under the terms of +the Less GNU General Public License (LGPL). + +Please visit the FastDFS Home Page for more detail. +Google code (English language): http://code.google.com/p/fastdfs/ +Chinese language: http://www.csource.com/ + +In file fastdfs_client.ini, item fastdfs_client.tracker_group# point to +the FastDFS client config filename. Please read ../INSTALL file to know +about how to config FastDFS client. + +FastDFS client php extension compiled under PHP 5.2.x, Steps: +phpize +./configure +make +make install + +#copy lib file to php extension directory, eg. /usr/lib/php/20060613/ +cp modules/fastdfs_client.so /usr/lib/php/20060613/ + +#copy fastdfs_client.ini to PHP etc directory, eg. /etc/php/ +cp fastdfs_client.ini /etc/php/ + +#modify config file fastdfs_client.ini, such as: +vi /etc/php/fastdfs_client.ini + +#run fastdfs_test.php +php fastdfs_test.php + + +FastDFS PHP functions: + +string fastdfs_client_version() +return client library version + + +long fastdfs_get_last_error_no() +return last error no + + +string fastdfs_get_last_error_info() +return last error info + + +string fastdfs_http_gen_token(string remote_filename, int timestamp) +generate anti-steal token for HTTP download +parameters: + remote_filename: the remote filename (do NOT including group name) + timestamp: the timestamp (unix timestamp) +return token string for success, false for error + + +array fastdfs_get_file_info(string group_name, string filename) +get file info from the filename +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server +return assoc array for success, false for error. + the assoc array including following elements: + create_timestamp: the file create timestamp (unix timestamp) + file_size: the file size (bytes) + source_ip_addr: the source storage server ip address + + +array fastdfs_get_file_info1(string file_id) +get file info from the file id +parameters: + file_id: the file id (including group name and filename) or remote filename +return assoc array for success, false for error. + the assoc array including following elements: + create_timestamp: the file create timestamp (unix timestamp) + file_size: the file size (bytes) + source_ip_addr: the source storage server ip address + + +bool fastdfs_send_data(int sock, string buff) +parameters: + sock: the unix socket description + buff: the buff to send +return true for success, false for error + + +string fastdfs_gen_slave_filename(string master_filename, string prefix_name + [, string file_ext_name]) +generate slave filename by master filename, prefix name and file extension name +parameters: + master_filename: the master filename / file id to generate + the slave filename + prefix_name: the prefix name to generate the slave filename + file_ext_name: slave file extension name, can be null or emtpy + (do not including dot) +return slave filename string for success, false for error + + +boolean fastdfs_storage_file_exist(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +check file exist +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for exist, false for not exist + + +boolean fastdfs_storage_file_exist1(string file_id + [, array tracker_server, array storage_server]) +parameters: + file_id: the file id of the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for exist, false for not exist + + +array fastdfs_storage_upload_by_filename(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_by_filename1(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error. + + +array fastdfs_storage_upload_by_filebuff(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_by_filebuff1(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array fastdfs_storage_upload_by_callback(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +array fastdfs_storage_upload_by_callback1(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array fastdfs_storage_upload_appender_by_filename(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server as appender file +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_appender_by_filename1(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server as appender file +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error. + + +array fastdfs_storage_upload_appender_by_filebuff(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server as appender file +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_appender_by_filebuff1(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server as appender file +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array fastdfs_storage_upload_appender_by_callback(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback as appender file +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_appender_by_callback1(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback as appender file +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +boolean fastdfs_storage_append_by_filename(string local_filename, + string group_name, appender_filename + [, array tracker_server, array storage_server]) +append local file to the appender file of storage server +parameters: + local_filename: the local filename + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +string fastdfs_storage_append_by_filename1(string local_filename, + string appender_file_id [, array tracker_server, array storage_server]) +append local file to the appender file of storage server +parameters: + local_filename: the local filename + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_append_by_filebuff(string file_buff, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +append file buff to the appender file of storage server +parameters: + file_buff: the file content + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_append_by_filebuff1(string file_buff, + string appender_file_id [, array tracker_server, array storage_server]) +append file buff to the appender file of storage server +parameters: + file_buff: the file content + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_append_by_callback(array callback_array, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +append file to the appender file of storage server by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_append_by_callback1(array callback_array, + string appender_file_id [, array tracker_server, array storage_server]) +append file buff to the appender file of storage server +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_modify_by_filename(string local_filename, + long file_offset, string group_name, appender_filename, + [array tracker_server, array storage_server]) +modify appender file by local file +parameters: + local_filename: the local filename + file_offset: offset of appender file + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_modify_by_filename1(string local_filename, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +modify appender file by local file +parameters: + local_filename: the local filename + file_offset: offset of appender file + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_modify_by_filebuff(string file_buff, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +modify appender file by file buff +parameters: + file_buff: the file content + file_offset: offset of appender file + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_modify_by_filebuff1(string file_buff, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +modify appender file by file buff +parameters: + file_buff: the file content + file_offset: offset of appender file + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_modify_by_callback(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +modify appender file by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_offset: offset of appender file + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_modify_by_callback1(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +modify appender file by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_offset: offset of appender file + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_truncate_file(string group_name, + string appender_filename [, long truncated_file_size = 0, + array tracker_server, array storage_server]) +truncate appender file to specify size +parameters: + group_name: the the group name of appender file + appender_filename: the appender filename + truncated_file_size: truncate the file size to + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean fastdfs_storage_truncate_file1(string appender_file_id + [, long truncated_file_size = 0, array tracker_server, + array storage_server]) +truncate appender file to specify size +parameters: + appender_file_id: the appender file id + truncated_file_size: truncate the file size to + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +string/array fastdfs_storage_upload_slave_by_filename(string local_filename, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload local file to storage server (slave file mode) +parameters: + file_buff: the file content + group_name: the group name of the master file + master_filename: the master filename to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_slave_by_filename1(string local_filename, + string master_file_id, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload local file to storage server (slave file mode) +parameters: + local_filename: the local filename + master_file_id: the master file id to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error. + + +array fastdfs_storage_upload_slave_by_filebuff(string file_buff, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file buff to storage server (slave file mode) +parameters: + file_buff: the file content + group_name: the group name of the master file + master_filename: the master filename to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_slave_by_filebuff1(string file_buff, + string master_file_id, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file buff to storage server (slave file mode) +parameters: + file_buff: the file content + master_file_id: the master file id to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array fastdfs_storage_upload_slave_by_callback(array callback_array, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file to storage server by callback (slave file mode) +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + group_name: the group name of the master file + master_filename: the master filename to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string fastdfs_storage_upload_slave_by_callback1(array callback_array, + string master_file_id, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file to storage server by callback (slave file mode) +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + master_file_id: the master file id to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +boolean fastdfs_storage_delete_file(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +delete file from storage server +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_storage_delete_file1(string file_id + [, array tracker_server, array storage_server]) +delete file from storage server +parameters: + file_id: the file id to be deleted + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +string fastdfs_storage_download_file_to_buff(string group_name, + string remote_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +get file content from storage server +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return the file content for success, false for error + + +string fastdfs_storage_download_file_to_buff1(string file_id + [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +get file content from storage server +parameters: + file_id: the file id of the file + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return the file content for success, false for error + + +boolean fastdfs_storage_download_file_to_file(string group_name, + string remote_filename, string local_filename [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +download file from storage server to local file +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + local_filename: the local filename to save the file content + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_storage_download_file_to_file1(string file_id, + string local_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +download file from storage server to local file +parameters: + file_id: the file id of the file + local_filename: the local filename to save the file content + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_storage_download_file_to_callback(string group_name, + string remote_filename, array download_callback [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + download_callback: the download callback array, elements as: + callback - the php callback function name + callback function prototype as: + function my_download_file_callback($args, $file_size, $data) + args - use argument for callback function + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_storage_download_file_to_callback1(string file_id, + array download_callback [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +parameters: + file_id: the file id of the file + download_callback: the download callback array, elements as: + callback - the php callback function name + callback function prototype as: + function my_download_file_callback($args, $file_size, $data) + args - use argument for callback function + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_storage_set_metadata(string group_name, string remote_filename, + array meta_list [, string op_type, array tracker_server, + array storage_server]) +set meta data of the file +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + meta_list: meta data assoc array to be set, such as + array('width'=>1024, 'height'=>768) + op_type: operate flag, can be one of following flags: + FDFS_STORAGE_SET_METADATA_FLAG_MERGE: combined with the old meta data + FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE: overwrite the old meta data + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_storage_set_metadata1(string file_id, array meta_list + [, string op_type, array tracker_server, array storage_server]) +set meta data of the file +parameters: + file_id: the file id of the file + meta_list: meta data assoc array to be set, such as + array('width'=>1024, 'height'=>768) + op_type: operate flag, can be one of following flags: + FDFS_STORAGE_SET_METADATA_FLAG_MERGE: combined with the old meta data + FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE: overwrite the old meta data + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +array fastdfs_storage_get_metadata(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +get meta data of the file +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + returned array like: array('width' => 1024, 'height' => 768) + + +array fastdfs_storage_get_metadata1(string file_id + [, array tracker_server, array storage_server]) +get meta data of the file +parameters: + file_id: the file id of the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + returned array like: array('width' => 1024, 'height' => 768) + + +array fastdfs_connect_server(string ip_addr, int port) +connect to the server +parameters: + ip_addr: the ip address of the server + port: the port of the server +return assoc array for success, false for error + + +boolean fastdfs_disconnect_server(array server_info) +disconnect from the server +parameters: + server_info: the assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean fastdfs_active_test(array server_info) +send ACTIVE_TEST cmd to the server +parameters: + server_info: the assoc array including elements: + ip_addr, port and sock, sock must be connected +return true for success, false for error + + +array fastdfs_tracker_get_connection() +get a connected tracker server +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +boolean fastdfs_tracker_make_all_connections() +connect to all tracker servers +return true for success, false for error + + +boolean fastdfs_tracker_close_all_connections() +close all connections to the tracker servers +return true for success, false for error + + +array fastdfs_tracker_list_groups([string group_name, array tracker_server]) +get group stat info +parameters: + group_name: specify the group name, null or empty string means all groups + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return index array for success, false for error, each group as a array element + + +array fastdfs_tracker_query_storage_store([string group_name, + array tracker_server]) +get the storage server info to upload file +parameters: + group_name: specify the group name + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. the assoc array including + elements: ip_addr, port, sock and store_path_index + + +array fastdfs_tracker_query_storage_store_list([string group_name, + array tracker_server]) +get the storage server list to upload file +parameters: + group_name: specify the group name + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return indexed storage server array for success, false for error. + each element is an ssoc array including elements: + ip_addr, port, sock and store_path_index + + +array fastdfs_tracker_query_storage_update(string group_name, + string remote_filename [, array tracker_server]) +get the storage server info to set metadata +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +array fastdfs_tracker_query_storage_update1(string file_id, + [, array tracker_server]) +get the storage server info to set metadata +parameters: + file_id: the file id of the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +array fastdfs_tracker_query_storage_fetch(string group_name, + string remote_filename [, array tracker_server]) +get the storage server info to download file (or get metadata) +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + +array fastdfs_tracker_query_storage_fetch1(string file_id + [, array tracker_server]) +get the storage server info to download file (or get metadata) +parameters: + file_id: the file id of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +array fastdfs_tracker_query_storage_list(string group_name, + string remote_filename [, array tracker_server]) +get the storage server list which can retrieve the file content or metadata +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return index array for success, false for error. + each server as an array element + + +array fastdfs_tracker_query_storage_list1(string file_id + [, array tracker_server]) +get the storage server list which can retrieve the file content or metadata +parameters: + file_id: the file id of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return index array for success, false for error. + each server as an array element + + +boolean fastdfs_tracker_delete_storage(string group_name, string storage_ip) +delete the storage server from the cluster +parameters: + group_name: the group name of the storage server + storage_ip: the ip address of the storage server to be deleted +return true for success, false for error + + +FastDFS Class Info: + +class FastDFS([int config_index, boolean bMultiThread]); +FastDFS class constructor +params: + config_index: use which config file, base 0. default is 0 + bMultiThread: if in multi-thread, default is false + + +long FastDFS::get_last_error_no() +return last error no + + +string FastDFS::get_last_error_info() +return last error info + +bool FastDFS::send_data(int sock, string buff) +parameters: + sock: the unix socket description + buff: the buff to send +return true for success, false for error + + +string FastDFS::http_gen_token(string remote_filename, int timestamp) +generate anti-steal token for HTTP download +parameters: + remote_filename: the remote filename (do NOT including group name) + timestamp: the timestamp (unix timestamp) +return token string for success, false for error + + +array FastDFS::get_file_info(string group_name, string filename) +get file info from the filename +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server +return assoc array for success, false for error. + the assoc array including following elements: + create_timestamp: the file create timestamp (unix timestamp) + file_size: the file size (bytes) + source_ip_addr: the source storage server ip address + crc32: the crc32 signature of the file + + +array FastDFS::get_file_info1(string file_id) +get file info from the file id +parameters: + file_id: the file id (including group name and filename) or remote filename +return assoc array for success, false for error. + the assoc array including following elements: + create_timestamp: the file create timestamp (unix timestamp) + file_size: the file size (bytes) + source_ip_addr: the source storage server ip address + + +string FastDFS::gen_slave_filename(string master_filename, string prefix_name + [, string file_ext_name]) +generate slave filename by master filename, prefix name and file extension name +parameters: + master_filename: the master filename / file id to generate + the slave filename + prefix_name: the prefix name to generate the slave filename + file_ext_name: slave file extension name, can be null or emtpy + (do not including dot) +return slave filename string for success, false for error + + +boolean FastDFS::storage_file_exist(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +check file exist +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for exist, false for not exist + + +boolean FastDFS::storage_file_exist1(string file_id + [, array tracker_server, array storage_server]) +parameters: + file_id: the file id of the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for exist, false for not exist + + +array FastDFS::storage_upload_by_filename(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_by_filename1(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error. + + +array FastDFS::storage_upload_by_filebuff(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_by_filebuff1(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array FastDFS::storage_upload_by_callback(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +array FastDFS::storage_upload_by_callback1(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array FastDFS::storage_upload_appender_by_filename(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server as appender file +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_appender_by_filename1(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload local file to storage server as appender file +parameters: + local_filename: the local filename + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error. + + +array FastDFS::storage_upload_appender_by_filebuff(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server as appender file +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_appender_by_filebuff1(string file_buff + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file buff to storage server as appender file +parameters: + file_buff: the file content + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array FastDFS::storage_upload_appender_by_callback(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback as appender file +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_appender_by_callback1(array callback_array + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +upload file to storage server by callback as appender file +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + group_name: specify the group name to store the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +boolean FastDFS::storage_append_by_filename(string local_filename, + string group_name, appender_filename + [, array tracker_server, array storage_server]) +append local file to the appender file of storage server +parameters: + local_filename: the local filename + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +string FastDFS::storage_upload_by_filename1(string local_filename, + [string appender_file_id, array tracker_server, array storage_server]) +append local file to the appender file of storage server +parameters: + local_filename: the local filename + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_append_by_filebuff(string file_buff, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +append file buff to the appender file of storage server +parameters: + file_buff: the file content + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_append_by_filebuff1(string file_buff, + string appender_file_id [, array tracker_server, array storage_server]) +append file buff to the appender file of storage server +parameters: + file_buff: the file content + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_append_by_callback(array callback_array, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +append file to the appender file of storage server by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_append_by_callback1(array callback_array, + string appender_file_id [, array tracker_server, array storage_server]) +append file buff to the appender file of storage server +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_modify_by_filename(string local_filename, + long file_offset, string group_name, appender_filename, + [array tracker_server, array storage_server]) +modify appender file by local file +parameters: + local_filename: the local filename + file_offset: offset of appender file + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_modify_by_filename1(string local_filename, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +modify appender file by local file +parameters: + local_filename: the local filename + file_offset: offset of appender file + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_modify_by_filebuff(string file_buff, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +modify appender file by file buff +parameters: + file_buff: the file content + file_offset: offset of appender file + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_modify_by_filebuff1(string file_buff, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +modify appender file by file buff +parameters: + file_buff: the file content + file_offset: offset of appender file + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_modify_by_callback(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +modify appender file by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_offset: offset of appender file + group_name: the the group name of appender file + appender_filename: the appender filename + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_modify_by_callback1(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +modify appender file by callback +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + file_offset: offset of appender file + appender_file_id: the appender file id + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_truncate_file(string group_name, + string appender_filename [, long truncated_file_size = 0, + array tracker_server, array storage_server]) +truncate appender file to specify size +parameters: + group_name: the the group name of appender file + appender_filename: the appender filename + truncated_file_size: truncate the file size to + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +boolean FastDFS::storage_truncate_file1(string appender_file_id + [, long truncated_file_size = 0, array tracker_server, + array storage_server]) +truncate appender file to specify size +parameters: + appender_file_id: the appender file id + truncated_file_size: truncate the file size to + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + + +array FastDFS::storage_upload_slave_by_filename(string local_filename, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload local file to storage server (slave file mode) +parameters: + file_buff: the file content + group_name: the group name of the master file + master_filename: the master filename to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_slave_by_filename1(string local_filename, + string master_file_id, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload local file to storage server (slave file mode) +parameters: + local_filename: the local filename + master_file_id: the master file id to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error. + + +array FastDFS::storage_upload_slave_by_filebuff(string file_buff, + string group_name, string master_filename, string prefix_name + [, file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file buff to storage server (slave file mode) +parameters: + file_buff: the file content + group_name: the group name of the master file + master_filename: the master filename to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_slave_by_filebuff1(string file_buff, + string master_file_id, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file buff to storage server (slave file mode) +parameters: + file_buff: the file content + master_file_id: the master file id to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +array FastDFS::storage_upload_slave_by_callback(array callback_array, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file to storage server by callback (slave file mode) +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + group_name: the group name of the master file + master_filename: the master filename to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. + the returned array includes elements: group_name and filename + + +string FastDFS::storage_upload_slave_by_callback1(array callback_array, + string master_file_id, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +upload file to storage server by callback (slave file mode) +parameters: + callback_array: the callback assoc array, must have keys: + callback - the php callback function name + callback function prototype as: + function upload_file_callback($sock, $args) + file_size - the file size + args - use argument for callback function + master_file_id: the master file id to generate the slave file id + prefix_name: the prefix name to generage the slave file id + file_ext_name: the file extension name, do not include dot(.) + meta_list: meta data assoc array, such as + array('width'=>1024, 'height'=>768) + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return file_id for success, false for error + + +boolean FastDFS::storage_delete_file(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +delete file from storage server +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean FastDFS::storage_delete_file1(string file_id + [, array tracker_server, array storage_server]) +delete file from storage server +parameters: + file_id: the file id to be deleted + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +string FastDFS::storage_download_file_to_buff(string group_name, + string remote_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +get file content from storage server +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return the file content for success, false for error + + +string FastDFS::storage_download_file_to_buff1(string file_id + [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +get file content from storage server +parameters: + file_id: the file id of the file + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return the file content for success, false for error + + +boolean FastDFS::storage_download_file_to_file(string group_name, + string remote_filename, string local_filename [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +download file from storage server to local file +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + local_filename: the local filename to save the file content + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean FastDFS::storage_download_file_to_file1(string file_id, + string local_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +download file from storage server to local file +parameters: + file_id: the file id of the file + local_filename: the local filename to save the file content + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean FastDFS::storage_download_file_to_callback(string group_name, + string remote_filename, array download_callback [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + download_callback: the download callback array, elements as: + callback - the php callback function name + callback function prototype as: + function my_download_file_callback($args, $file_size, $data) + args - use argument for callback function + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean FastDFS::storage_download_file_to_callback1(string file_id, + array download_callback [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +parameters: + file_id: the file id of the file + download_callback: the download callback array, elements as: + callback - the php callback function name + callback function prototype as: + function my_download_file_callback($args, $file_size, $data) + args - use argument for callback function + file_offset: file start offset, default value is 0 + download_bytes: 0 (default value) means from the file offset to + the file end + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean FastDFS::storage_set_metadata(string group_name, string remote_filename, + array meta_list [, string op_type, array tracker_server, + array storage_server]) +set meta data of the file +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + meta_list: meta data assoc array to be set, such as + array('width'=>1024, 'height'=>768) + op_type: operate flag, can be one of following flags: + FDFS_STORAGE_SET_METADATA_FLAG_MERGE: combined with the old meta data + FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE: overwrite the old meta data + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +boolean FastDFS::storage_set_metadata1(string file_id, array meta_list + [, string op_type, array tracker_server, array storage_server]) +set meta data of the file +parameters: + file_id: the file id of the file + meta_list: meta data assoc array to be set, such as + array('width'=>1024, 'height'=>768) + op_type: operate flag, can be one of following flags: + FDFS_STORAGE_SET_METADATA_FLAG_MERGE: combined with the old meta data + FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE: overwrite the old meta data + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +array FastDFS::storage_get_metadata(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +get meta data of the file +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + returned array like: array('width' => 1024, 'height' => 768) + + +array FastDFS::storage_get_metadata1(string file_id + [, array tracker_server, array storage_server]) +get meta data of the file +parameters: + file_id: the file id of the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock + storage_server: the storage server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + returned array like: array('width' => 1024, 'height' => 768) + + +array FastDFS::connect_server(string ip_addr, int port) +connect to the server +parameters: + ip_addr: the ip address of the server + port: the port of the server +return assoc array for success, false for error + + +boolean FastDFS::disconnect_server(array server_info) +disconnect from the server +parameters: + server_info: the assoc array including elements: + ip_addr, port and sock +return true for success, false for error + + +array FastDFS::tracker_get_connection() +get a connected tracker server +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +boolean FastDFS::active_test(array server_info) +send ACTIVE_TEST cmd to the server +parameters: + server_info: the assoc array including elements: + ip_addr, port and sock, sock must be connected +return true for success, false for error + + +boolean FastDFS::tracker_make_all_connections() +connect to all tracker servers +return true for success, false for error + + +boolean FastDFS::tracker_close_all_connections() +close all connections to the tracker servers +return true for success, false for error + + +array FastDFS::tracker_list_groups([string group_name, array tracker_server]) +get group stat info +parameters: + group_name: specify the group name, null or empty string means all groups + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return index array for success, false for error, each group as a array element + + +array FastDFS::tracker_query_storage_store([string group_name, + array tracker_server]) +get the storage server info to upload file +parameters: + group_name: specify the group name + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error. the assoc array including + elements: ip_addr, port, sock and store_path_index + + +array FastDFS::tracker_query_storage_store_list([string group_name, + array tracker_server]) +get the storage server list to upload file +parameters: + group_name: specify the group name + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return indexed storage server array for success, false for error. + each element is an ssoc array including elements: + ip_addr, port, sock and store_path_index + + +array FastDFS::tracker_query_storage_update(string group_name, + string remote_filename [, array tracker_server]) +get the storage server info to set metadata +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +array FastDFS::tracker_query_storage_update1(string file_id, + [, array tracker_server]) +get the storage server info to set metadata +parameters: + file_id: the file id of the file + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +array FastDFS::tracker_query_storage_fetch(string group_name, + string remote_filename [, array tracker_server]) +get the storage server info to download file (or get metadata) +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + +array FastDFS::tracker_query_storage_fetch1(string file_id + [, array tracker_server]) +get the storage server info to download file (or get metadata) +parameters: + file_id: the file id of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return assoc array for success, false for error + the assoc array including elements: ip_addr, port and sock + + +array FastDFS::tracker_query_storage_list(string group_name, + string remote_filename [, array tracker_server]) +get the storage server list which can retrieve the file content or metadata +parameters: + group_name: the group name of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return index array for success, false for error. + each server as an array element + + +array FastDFS::tracker_query_storage_list1(string file_id + [, array tracker_server]) +get the storage server list which can retrieve the file content or metadata +parameters: + file_id: the file id of the file + remote_filename: the filename on the storage server + tracker_server: the tracker server assoc array including elements: + ip_addr, port and sock +return index array for success, false for error. + each server as an array element + + +boolean FastDFS::tracker_delete_storage(string group_name, string storage_ip) +delete the storage server from the cluster +parameters: + group_name: the group name of the storage server + storage_ip: the ip address of the storage server to be deleted +return true for success, false for error + +void FastDFS::close() +close tracker connections + diff --git a/php_client/config.m4 b/php_client/config.m4 new file mode 100644 index 0000000..171346b --- /dev/null +++ b/php_client/config.m4 @@ -0,0 +1,22 @@ +dnl config.m4 for extension fastdfs_client + +PHP_ARG_WITH(fastdfs_client, for fastdfs_client support FastDFS client, +[ --with-fastdfs_client Include fastdfs_client support FastDFS client]) + +if test "$PHP_FASTDFS_CLIENT" != "no"; then + PHP_SUBST(FASTDFS_CLIENT_SHARED_LIBADD) + + if test -z "$ROOT"; then + ROOT=/usr/local + fi + + PHP_ADD_INCLUDE($ROOT/include/fastcommon) + PHP_ADD_INCLUDE($ROOT/include/fastdfs) + + PHP_ADD_LIBRARY_WITH_PATH(fastcommon, $ROOT/lib, FASTDFS_CLIENT_SHARED_LIBADD) + PHP_ADD_LIBRARY_WITH_PATH(fdfsclient, $ROOT/lib, FASTDFS_CLIENT_SHARED_LIBADD) + + PHP_NEW_EXTENSION(fastdfs_client, fastdfs_client.c, $ext_shared) + + CFLAGS="$CFLAGS -Wall" +fi diff --git a/php_client/fastdfs_appender_test.php b/php_client/fastdfs_appender_test.php new file mode 100644 index 0000000..00653b4 --- /dev/null +++ b/php_client/fastdfs_appender_test.php @@ -0,0 +1,88 @@ +storage_upload_appender_by_filename("/usr/include/stdio.h"); + if (!$file_info) + { + echo "$fdfs->storage_upload_appender_by_filename fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + $file_id = "$group_name/$remote_filename"; + var_dump($fdfs->get_file_info($group_name, $remote_filename)); + + $appender_filename = $remote_filename; + echo "file id: $group_name/$appender_filename\n"; + if (!$fdfs->storage_append_by_filename("/usr/include/stdlib.h", $group_name, $appender_filename)) + { + echo "$fdfs->storage_append_by_filename fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($fdfs->get_file_info($group_name, $appender_filename)); + + if (!$fdfs->storage_modify_by_filename("/usr/include/stdlib.h", 0, $group_name, $appender_filename)) + { + echo "$fdfs->storage_modify_by_filename fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($fdfs->get_file_info($group_name, $appender_filename)); + + if (!$fdfs->storage_truncate_file($group_name, $appender_filename)) + { + echo "$fdfs->storage_truncate_file fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($fdfs->get_file_info($group_name, $appender_filename)); + + echo 'tracker_close_all_connections result: ' . $fdfs->tracker_close_all_connections() . "\n"; +?> diff --git a/php_client/fastdfs_appender_test1.php b/php_client/fastdfs_appender_test1.php new file mode 100644 index 0000000..107180b --- /dev/null +++ b/php_client/fastdfs_appender_test1.php @@ -0,0 +1,76 @@ +storage_upload_appender_by_filename1("/usr/include/stdio.h"); + if (!$appender_file_id) + { + echo "\$fdfs->storage_upload_appender_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($appender_file_id); + var_dump($fdfs->get_file_info1($appender_file_id)); + + if (!$fdfs->storage_append_by_filename1("/usr/include/stdlib.h", $appender_file_id)) + { + echo "\$fdfs->storage_append_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($fdfs->get_file_info1($appender_file_id)); + + if (!$fdfs->storage_modify_by_filename1("/usr/include/stdlib.h", 0, $appender_file_id)) + { + echo "\$fdfs->storage_modify_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($fdfs->get_file_info1($appender_file_id)); + + if (!$fdfs->storage_truncate_file1($appender_file_id)) + { + echo "\$fdfs->torage_truncate_file1 torage_modify_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + exit; + } + + var_dump($fdfs->get_file_info1($appender_file_id)); + + echo 'tracker_close_all_connections result: ' . $fdfs->tracker_close_all_connections() . "\n"; +?> diff --git a/php_client/fastdfs_callback_test.php b/php_client/fastdfs_callback_test.php new file mode 100644 index 0000000..6675b6a --- /dev/null +++ b/php_client/fastdfs_callback_test.php @@ -0,0 +1,120 @@ + FILE_BUFF); + $upload_callback_array = array( + 'callback' => 'my_upload_file_callback', + 'file_size' => strlen(FILE_BUFF), + 'args' => $upload_callback_arg); + + $download_callback_arg = array ( + 'filename' => '/tmp/out.txt', + 'write_bytes' => 0, + 'fhandle' => NULL + ); + $download_callback_array = array( + 'callback' => 'my_download_file_callback', + 'args' => &$download_callback_arg); + + $file_info = fastdfs_storage_upload_by_callback($upload_callback_array); + if ($file_info) + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + var_dump(fastdfs_get_file_info($group_name, $remote_filename)); + + fastdfs_storage_download_file_to_callback($group_name, $remote_filename, $download_callback_array); + } + else + { + echo "upload file fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + $file_id = fastdfs_storage_upload_by_callback1($upload_callback_array, 'txt'); + if ($file_id) + { + var_dump($file_id); + $download_callback_arg['filename'] = '/tmp/out1.txt'; + fastdfs_storage_download_file_to_callback1($file_id, $download_callback_array); + } + else + { + echo "upload file fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + $fdfs = new FastDFS(); + $file_info = $fdfs->storage_upload_by_callback($upload_callback_array, 'txt'); + if ($file_info) + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + var_dump($fdfs->get_file_info($group_name, $remote_filename)); + $download_callback_arg['filename'] = '/tmp/fdfs_out.txt'; + $fdfs->storage_download_file_to_callback($group_name, $remote_filename, $download_callback_array); + } + else + { + echo "upload file fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + $file_id = $fdfs->storage_upload_by_callback1($upload_callback_array, 'txt'); + if ($file_id) + { + var_dump($file_id); + $download_callback_arg['filename'] = '/tmp/fdfs_out1.txt'; + $fdfs->storage_download_file_to_callback1($file_id, $download_callback_array); + } + else + { + echo "upload file fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + function my_upload_file_callback($sock, $args) + { + //var_dump($args); + + return fastdfs_send_data($sock, $args['buff']); + } + + function my_download_file_callback($args, $file_size, $data) + { + var_dump($args); + + if ($args['fhandle'] == NULL) + { + $args['fhandle'] = fopen ($args['filename'], 'w'); + if (!$args['fhandle']) + { + echo 'open file: ' . $args['filename'] . " fail!\n"; + return false; + } + } + + $len = strlen($data); + if (fwrite($args['fhandle'], $data, $len) === false) + { + echo 'write to file: ' . $args['filename'] . " fail!\n"; + $result = false; + } + else + { + $args['write_bytes'] += $len; + $result = true; + } + + if ((!$result) || $args['write_bytes'] >= $file_size) + { + fclose($args['fhandle']); + $args['fhandle'] = NULL; + $args['write_bytes'] = 0; + } + + return $result; + } +?> diff --git a/php_client/fastdfs_client.c b/php_client/fastdfs_client.c new file mode 100644 index 0000000..d884614 --- /dev/null +++ b/php_client/fastdfs_client.c @@ -0,0 +1,7512 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#ifdef ZTS +#include "TSRM.h" +#endif + +#include +#include +#include "ext/standard/info.h" +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_client.h" +#include "logger.h" +#include "sockopt.h" +#include "fdfs_global.h" +#include "shared_func.h" +#include "client_global.h" +#include "fastdfs_client.h" +#include "fdfs_http_shared.h" + +typedef struct +{ + TrackerServerGroup *pTrackerGroup; +} FDFSConfigInfo; + +typedef struct +{ + TrackerServerGroup *pTrackerGroup; + int err_no; +} FDFSPhpContext; + +typedef struct +{ + zend_object zo; + FDFSConfigInfo *pConfigInfo; + FDFSPhpContext context; +} php_fdfs_t; + +typedef struct +{ + zval *func_name; + zval *args; +} php_fdfs_callback_t; + +typedef struct +{ + php_fdfs_callback_t callback; + int64_t file_size; +} php_fdfs_upload_callback_t; + +static int php_fdfs_download_callback(void *arg, const int64_t file_size, \ + const char *data, const int current_size); + +static FDFSConfigInfo *config_list = NULL; +static int config_count = 0; + +static FDFSPhpContext php_context = {&g_tracker_group, 0}; + +static int le_fdht; + +static zend_class_entry *fdfs_ce = NULL; +static zend_class_entry *fdfs_exception_ce = NULL; + +#if HAVE_SPL +static zend_class_entry *spl_ce_RuntimeException = NULL; +#endif + +#if (PHP_MAJOR_VERSION == 5 && PHP_MINOR_VERSION < 3) +const zend_fcall_info empty_fcall_info = { 0, NULL, NULL, NULL, NULL, 0, NULL, NULL, 0 }; +#undef ZEND_BEGIN_ARG_INFO_EX +#define ZEND_BEGIN_ARG_INFO_EX(name, pass_rest_by_reference, return_reference, required_num_args) \ + static zend_arg_info name[] = { \ + { NULL, 0, NULL, 0, 0, 0, pass_rest_by_reference, return_reference, required_num_args }, +#endif + + +#define CLEAR_HASH_SOCK_FIELD(php_hash) \ + { \ + zval *sock_zval; \ + MAKE_STD_ZVAL(sock_zval); \ + ZVAL_LONG(sock_zval, -1); \ + \ + zend_hash_update(php_hash, "sock", sizeof("sock"), \ + &sock_zval, sizeof(zval *), NULL); \ + } + +// Every user visible function must have an entry in fastdfs_client_functions[]. + zend_function_entry fastdfs_client_functions[] = { + ZEND_FE(fastdfs_client_version, NULL) + ZEND_FE(fastdfs_active_test, NULL) + ZEND_FE(fastdfs_connect_server, NULL) + ZEND_FE(fastdfs_disconnect_server, NULL) + ZEND_FE(fastdfs_get_last_error_no, NULL) + ZEND_FE(fastdfs_get_last_error_info, NULL) + ZEND_FE(fastdfs_tracker_get_connection, NULL) + ZEND_FE(fastdfs_tracker_make_all_connections, NULL) + ZEND_FE(fastdfs_tracker_close_all_connections, NULL) + ZEND_FE(fastdfs_tracker_list_groups, NULL) + ZEND_FE(fastdfs_tracker_query_storage_store, NULL) + ZEND_FE(fastdfs_tracker_query_storage_store_list, NULL) + ZEND_FE(fastdfs_tracker_query_storage_update, NULL) + ZEND_FE(fastdfs_tracker_query_storage_fetch, NULL) + ZEND_FE(fastdfs_tracker_query_storage_list, NULL) + ZEND_FE(fastdfs_tracker_query_storage_update1, NULL) + ZEND_FE(fastdfs_tracker_query_storage_fetch1, NULL) + ZEND_FE(fastdfs_tracker_query_storage_list1, NULL) + ZEND_FE(fastdfs_tracker_delete_storage, NULL) + ZEND_FE(fastdfs_storage_upload_by_filename, NULL) + ZEND_FE(fastdfs_storage_upload_by_filename1, NULL) + ZEND_FE(fastdfs_storage_upload_by_filebuff, NULL) + ZEND_FE(fastdfs_storage_upload_by_filebuff1, NULL) + ZEND_FE(fastdfs_storage_upload_by_callback, NULL) + ZEND_FE(fastdfs_storage_upload_by_callback1, NULL) + ZEND_FE(fastdfs_storage_append_by_filename, NULL) + ZEND_FE(fastdfs_storage_append_by_filename1, NULL) + ZEND_FE(fastdfs_storage_append_by_filebuff, NULL) + ZEND_FE(fastdfs_storage_append_by_filebuff1, NULL) + ZEND_FE(fastdfs_storage_append_by_callback, NULL) + ZEND_FE(fastdfs_storage_append_by_callback1, NULL) + ZEND_FE(fastdfs_storage_modify_by_filename, NULL) + ZEND_FE(fastdfs_storage_modify_by_filename1, NULL) + ZEND_FE(fastdfs_storage_modify_by_filebuff, NULL) + ZEND_FE(fastdfs_storage_modify_by_filebuff1, NULL) + ZEND_FE(fastdfs_storage_modify_by_callback, NULL) + ZEND_FE(fastdfs_storage_modify_by_callback1, NULL) + ZEND_FE(fastdfs_storage_upload_appender_by_filename, NULL) + ZEND_FE(fastdfs_storage_upload_appender_by_filename1, NULL) + ZEND_FE(fastdfs_storage_upload_appender_by_filebuff, NULL) + ZEND_FE(fastdfs_storage_upload_appender_by_filebuff1, NULL) + ZEND_FE(fastdfs_storage_upload_appender_by_callback, NULL) + ZEND_FE(fastdfs_storage_upload_appender_by_callback1, NULL) + ZEND_FE(fastdfs_storage_upload_slave_by_filename, NULL) + ZEND_FE(fastdfs_storage_upload_slave_by_filename1, NULL) + ZEND_FE(fastdfs_storage_upload_slave_by_filebuff, NULL) + ZEND_FE(fastdfs_storage_upload_slave_by_filebuff1, NULL) + ZEND_FE(fastdfs_storage_upload_slave_by_callback, NULL) + ZEND_FE(fastdfs_storage_upload_slave_by_callback1, NULL) + ZEND_FE(fastdfs_storage_delete_file, NULL) + ZEND_FE(fastdfs_storage_delete_file1, NULL) + ZEND_FE(fastdfs_storage_truncate_file, NULL) + ZEND_FE(fastdfs_storage_truncate_file1, NULL) + ZEND_FE(fastdfs_storage_download_file_to_buff, NULL) + ZEND_FE(fastdfs_storage_download_file_to_buff1, NULL) + ZEND_FE(fastdfs_storage_download_file_to_file, NULL) + ZEND_FE(fastdfs_storage_download_file_to_file1, NULL) + ZEND_FE(fastdfs_storage_download_file_to_callback, NULL) + ZEND_FE(fastdfs_storage_download_file_to_callback1, NULL) + ZEND_FE(fastdfs_storage_set_metadata, NULL) + ZEND_FE(fastdfs_storage_set_metadata1, NULL) + ZEND_FE(fastdfs_storage_get_metadata, NULL) + ZEND_FE(fastdfs_storage_get_metadata1, NULL) + ZEND_FE(fastdfs_http_gen_token, NULL) + ZEND_FE(fastdfs_get_file_info, NULL) + ZEND_FE(fastdfs_get_file_info1, NULL) + ZEND_FE(fastdfs_storage_file_exist, NULL) + ZEND_FE(fastdfs_storage_file_exist1, NULL) + ZEND_FE(fastdfs_gen_slave_filename, NULL) + ZEND_FE(fastdfs_send_data, NULL) + {NULL, NULL, NULL} /* Must be the last line */ + }; + + +zend_module_entry fastdfs_client_module_entry = { + STANDARD_MODULE_HEADER, + "fastdfs_client", + fastdfs_client_functions, + PHP_MINIT(fastdfs_client), + PHP_MSHUTDOWN(fastdfs_client), + NULL,//PHP_RINIT(fastdfs_client), + NULL,//PHP_RSHUTDOWN(fastdfs_client), + PHP_MINFO(fastdfs_client), + "1.00", + STANDARD_MODULE_PROPERTIES +}; + +#ifdef COMPILE_DL_FASTDFS_CLIENT + ZEND_GET_MODULE(fastdfs_client) +#endif + +static int fastdfs_convert_metadata_to_array(zval *metadata_obj, \ + FDFSMetaData **meta_list, int *meta_count) +{ + HashTable *meta_hash; + char *szKey; + char *szValue; + unsigned long index; + unsigned int key_len; + int value_len; + HashPosition pointer; + zval ***ppp; + zval **data; + FDFSMetaData *pMetaData; + + meta_hash = Z_ARRVAL_P(metadata_obj); + *meta_count = zend_hash_num_elements(meta_hash); + if (*meta_count == 0) + { + *meta_list = NULL; + return 0; + } + + *meta_list = (FDFSMetaData *)malloc(sizeof(FDFSMetaData)*(*meta_count)); + if (*meta_list == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(FDFSMetaData) * (*meta_count), \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + memset(*meta_list, 0, sizeof(FDFSMetaData) * (*meta_count)); + pMetaData = *meta_list; + ppp = &data; + for (zend_hash_internal_pointer_reset_ex(meta_hash, &pointer); \ + zend_hash_get_current_data_ex(meta_hash, (void **)ppp, &pointer) + == SUCCESS; zend_hash_move_forward_ex(meta_hash, &pointer)) + { + if (zend_hash_get_current_key_ex(meta_hash, &szKey, \ + &(key_len), &index, 0, &pointer) != HASH_KEY_IS_STRING) + { + logError("file: "__FILE__", line: %d, " \ + "invalid array element, " \ + "index=%ld!", __LINE__, index); + + free(*meta_list); + *meta_list = NULL; + *meta_count = 0; + return EINVAL; + } + + if (key_len > FDFS_MAX_META_NAME_LEN) + { + key_len = FDFS_MAX_META_NAME_LEN; + } + memcpy(pMetaData->name, szKey, key_len); + + if ((*data)->type == IS_STRING) + { + szValue = Z_STRVAL_PP(data); + value_len = Z_STRLEN_PP(data); + + if (value_len > FDFS_MAX_META_VALUE_LEN) + { + value_len = FDFS_MAX_META_VALUE_LEN; + } + memcpy(pMetaData->value, szValue, value_len); + } + else if ((*data)->type == IS_LONG || (*data)->type == IS_BOOL) + { + sprintf(pMetaData->value, "%ld", (*data)->value.lval); + } + else if ((*data)->type == IS_DOUBLE) + { + sprintf(pMetaData->value, "%.2f", (*data)->value.dval); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "invalid array element, key=%s, value type=%d",\ + __LINE__, szKey, (*data)->type); + + free(*meta_list); + *meta_list = NULL; + *meta_count = 0; + return EINVAL; + } + + pMetaData++; + } + + return 0; +} + +static void php_fdfs_tracker_get_connection_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + ConnectionInfo *pTrackerServer; + + argc = ZEND_NUM_ARGS(); + if (argc != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_get_connection parameters count: %d != 0", + __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + pTrackerServer = tracker_get_connection_no_pool(pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + + pContext->err_no = 0; + array_init(return_value); + + add_assoc_stringl_ex(return_value, "ip_addr", sizeof("ip_addr"), \ + pTrackerServer->ip_addr, strlen(pTrackerServer->ip_addr), 1); + add_assoc_long_ex(return_value, "port", sizeof("port"), \ + pTrackerServer->port); + add_assoc_long_ex(return_value, "sock", sizeof("sock"), \ + pTrackerServer->sock); +} + +static void php_fdfs_tracker_make_all_connections_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext) +{ + int argc; + + argc = ZEND_NUM_ARGS(); + if (argc != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_make_all_connections parameters " \ + "count: %d != 0", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + pContext->err_no = tracker_get_all_connections_ex( \ + pContext->pTrackerGroup); + if (pContext->err_no == 0) + { + RETURN_BOOL(true); + } + else + { + RETURN_BOOL(false); + } +} + +static void php_fdfs_tracker_close_all_connections_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext) +{ + int argc; + + argc = ZEND_NUM_ARGS(); + if (argc != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_close_all_connections parameters " \ + "count: %d != 0", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_close_all_connections_ex(pContext->pTrackerGroup); + pContext->err_no = 0; + RETURN_BOOL(true); +} + +static void php_fdfs_connect_server_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + char *ip_addr; + int ip_len; + long port; + ConnectionInfo server_info; + + argc = ZEND_NUM_ARGS(); + if (argc != 2) + { + logError("file: "__FILE__", line: %d, " \ + "fastdfs_connect_server parameters count: %d != 2", \ + __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl", \ + &ip_addr, &ip_len, &port) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(server_info.ip_addr, sizeof(server_info.ip_addr), \ + "%s", ip_addr); + server_info.port = port; + server_info.sock = -1; + + if ((pContext->err_no=conn_pool_connect_server(&server_info, \ + g_fdfs_network_timeout)) == 0) + { + array_init(return_value); + add_assoc_stringl_ex(return_value, "ip_addr", \ + sizeof("ip_addr"), ip_addr, ip_len, 1); + add_assoc_long_ex(return_value, "port", sizeof("port"), \ + port); + add_assoc_long_ex(return_value, "sock", sizeof("sock"), \ + server_info.sock); + } + else + { + RETURN_BOOL(false); + } +} + +static void php_fdfs_disconnect_server_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + zval *server_info; + HashTable *tracker_hash; + zval **data; + zval ***ppp; + int sock; + + argc = ZEND_NUM_ARGS(); + if (argc != 1) + { + logError("file: "__FILE__", line: %d, " \ + "fastdfs_disconnect_server parameters count: %d != 1", \ + __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "a", \ + &server_info) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_hash = Z_ARRVAL_P(server_info); + data = NULL; + ppp = &data; + if (zend_hash_find(tracker_hash, "sock", sizeof("sock"), \ + (void **)ppp) == FAILURE) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + + if ((*data)->type == IS_LONG) + { + sock = (*data)->value.lval; + if (sock >= 0) + { + close(sock); + } + + CLEAR_HASH_SOCK_FIELD(tracker_hash) + + pContext->err_no = 0; + RETURN_BOOL(true); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "sock type is invalid, type=%d!", \ + __LINE__, (*data)->type); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } +} + +static int php_fdfs_get_callback_from_hash(HashTable *callback_hash, \ + php_fdfs_callback_t *pCallback) +{ + zval **data; + zval ***ppp; + + data = NULL; + ppp = &data; + if (zend_hash_find(callback_hash, "callback", sizeof("callback"), \ + (void **)ppp) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "key \"callback\" not exist!", __LINE__); + return ENOENT; + } + if ((*data)->type != IS_STRING) + { + logError("file: "__FILE__", line: %d, " \ + "key \"callback\" is not string type, type=%d!", \ + __LINE__, (*data)->type); + return EINVAL; + } + pCallback->func_name = *data; + + data = NULL; + if (zend_hash_find(callback_hash, "args", sizeof("args"), \ + (void **)ppp) == FAILURE) + { + pCallback->args = NULL; + } + else + { + pCallback->args = ((*data)->type == IS_NULL) ? NULL : *data; + } + + return 0; +} + +static int php_fdfs_get_upload_callback_from_hash(HashTable *callback_hash, \ + php_fdfs_upload_callback_t *pUploadCallback) +{ + zval **data; + zval ***ppp; + int result; + + if ((result=php_fdfs_get_callback_from_hash(callback_hash, \ + &(pUploadCallback->callback))) != 0) + { + return result; + } + + data = NULL; + ppp = &data; + if (zend_hash_find(callback_hash, "file_size", sizeof("file_size"), \ + (void **)ppp) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "key \"file_size\" not exist!", __LINE__); + return ENOENT; + } + if ((*data)->type != IS_LONG) + { + logError("file: "__FILE__", line: %d, " \ + "key \"file_size\" is not long type, type=%d!", \ + __LINE__, (*data)->type); + return EINVAL; + } + pUploadCallback->file_size = (*data)->value.lval; + if (pUploadCallback->file_size < 0) + { + logError("file: "__FILE__", line: %d, " \ + "file_size: "INT64_PRINTF_FORMAT" is invalid!", \ + __LINE__, pUploadCallback->file_size); + return EINVAL; + } + + return 0; +} + +static int php_fdfs_get_server_from_hash(HashTable *tracker_hash, \ + ConnectionInfo *pTrackerServer) +{ + zval **data; + zval ***ppp; + char *ip_addr; + int ip_len; + + memset(pTrackerServer, 0, sizeof(ConnectionInfo)); + data = NULL; + ppp = &data; + if (zend_hash_find(tracker_hash, "ip_addr", sizeof("ip_addr"), \ + (void **)ppp) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "key \"ip_addr\" not exist!", __LINE__); + return ENOENT; + } + if ((*data)->type != IS_STRING) + { + logError("file: "__FILE__", line: %d, " \ + "key \"ip_addr\" is not string type, type=%d!", \ + __LINE__, (*data)->type); + return EINVAL; + } + + ip_addr = Z_STRVAL_PP(data); + ip_len = Z_STRLEN_PP(data); + if (ip_len >= IP_ADDRESS_SIZE) + { + ip_len = IP_ADDRESS_SIZE - 1; + } + memcpy(pTrackerServer->ip_addr, ip_addr, ip_len); + + if (zend_hash_find(tracker_hash, "port", sizeof("port"), \ + (void **)ppp) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "key \"port\" not exist!", __LINE__); + return ENOENT; + } + if ((*data)->type != IS_LONG) + { + logError("file: "__FILE__", line: %d, " \ + "key \"port\" is not long type, type=%d!", \ + __LINE__, (*data)->type); + return EINVAL; + } + pTrackerServer->port = (*data)->value.lval; + + if (zend_hash_find(tracker_hash, "sock", sizeof("sock"), \ + (void **)ppp) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "key \"sock\" not exist!", __LINE__); + return ENOENT; + } + if ((*data)->type != IS_LONG) + { + logError("file: "__FILE__", line: %d, " \ + "key \"sock\" is not long type, type=%d!", \ + __LINE__, (*data)->type); + return EINVAL; + } + + pTrackerServer->sock = (*data)->value.lval; + return 0; +} + +static void php_fastdfs_active_test_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + zval *server_info; + HashTable *tracker_hash; + ConnectionInfo server; + + argc = ZEND_NUM_ARGS(); + if (argc != 1) + { + logError("file: "__FILE__", line: %d, " \ + "fastdfs_active_test parameters count: %d != 1", \ + __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "a", \ + &server_info) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_hash = Z_ARRVAL_P(server_info); + + if ((pContext->err_no=php_fdfs_get_server_from_hash(tracker_hash, \ + &server)) != 0) + { + RETURN_BOOL(false); + } + + if ((pContext->err_no=fdfs_active_test(&server)) != 0) + { + RETURN_BOOL(false); + } + else + { + RETURN_BOOL(true); + } +} + +static void php_fdfs_tracker_list_groups_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + char *group_name; + int group_nlen; + zval *tracker_obj; + zval *group_info_array; + zval *server_info_array; + HashTable *tracker_hash; + ConnectionInfo tracker_server; + ConnectionInfo *pTrackerServer; + FDFSGroupStat group_stats[FDFS_MAX_GROUPS]; + FDFSGroupStat *pGroupStat; + FDFSGroupStat *pGroupEnd; + int group_count; + int result; + int storage_count; + int saved_tracker_sock; + FDFSStorageInfo storage_infos[FDFS_MAX_SERVERS_EACH_GROUP]; + FDFSStorageInfo *pStorage; + FDFSStorageInfo *pStorageEnd; + FDFSStorageStat *pStorageStat; + + argc = ZEND_NUM_ARGS(); + if (argc > 2) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_list_groups parameters count: %d > 2", + __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + group_name = NULL; + group_nlen = 0; + tracker_obj = NULL; + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|sa", \ + &group_name, &group_nlen, &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + saved_tracker_sock = pTrackerServer->sock; + } + + if (group_name != NULL && group_nlen > 0) + { + group_count = 1; + result = tracker_list_one_group(pTrackerServer, group_name, \ + group_stats); + } + else + { + result = tracker_list_groups(pTrackerServer, group_stats, \ + FDFS_MAX_GROUPS, &group_count); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + pContext->err_no = result; + RETURN_BOOL(false); + } + + pContext->err_no = 0; + array_init(return_value); + + pGroupEnd = group_stats + group_count; + for (pGroupStat=group_stats; pGroupStatgroup_name, \ + strlen(pGroupStat->group_name) + 1, group_info_array); + + add_assoc_long_ex(group_info_array, "total_space", \ + sizeof("total_space"), pGroupStat->total_mb); + add_assoc_long_ex(group_info_array, "free_space", \ + sizeof("free_space"), pGroupStat->free_mb); + add_assoc_long_ex(group_info_array, "trunk_free_space", \ + sizeof("trunk_free_space"), pGroupStat->trunk_free_mb); + add_assoc_long_ex(group_info_array, "server_count", \ + sizeof("server_count"), pGroupStat->count); + add_assoc_long_ex(group_info_array, "active_count", \ + sizeof("active_count"), pGroupStat->active_count); + add_assoc_long_ex(group_info_array, "storage_port", \ + sizeof("storage_port"), pGroupStat->storage_port); + add_assoc_long_ex(group_info_array, "storage_http_port", \ + sizeof("storage_http_port"), \ + pGroupStat->storage_http_port); + add_assoc_long_ex(group_info_array, "store_path_count", \ + sizeof("store_path_count"), \ + pGroupStat->store_path_count); + add_assoc_long_ex(group_info_array, "subdir_count_per_path", \ + sizeof("subdir_count_per_path"), \ + pGroupStat->subdir_count_per_path); + add_assoc_long_ex(group_info_array, "current_write_server", \ + sizeof("current_write_server"), \ + pGroupStat->current_write_server); + add_assoc_long_ex(group_info_array, "current_trunk_file_id", \ + sizeof("current_trunk_file_id"), \ + pGroupStat->current_trunk_file_id); + + result = tracker_list_servers(pTrackerServer, \ + pGroupStat->group_name, NULL, \ + storage_infos, FDFS_MAX_SERVERS_EACH_GROUP, \ + &storage_count); + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + + pContext->err_no = result; + RETURN_BOOL(false); + } + + pStorageEnd = storage_infos + storage_count; + for (pStorage=storage_infos; pStorageid, \ + strlen(pStorage->id) + 1, server_info_array); + + add_assoc_stringl_ex(server_info_array, \ + "ip_addr", sizeof("ip_addr"), \ + pStorage->ip_addr, strlen(pStorage->ip_addr), 1); + + add_assoc_long_ex(server_info_array, \ + "join_time", sizeof("join_time"), \ + pStorage->join_time); + + add_assoc_long_ex(server_info_array, \ + "up_time", sizeof("up_time"), \ + pStorage->up_time); + + add_assoc_stringl_ex(server_info_array, \ + "http_domain", sizeof("http_domain"), \ + pStorage->domain_name, \ + strlen(pStorage->domain_name), 1); + + add_assoc_stringl_ex(server_info_array, \ + "version", sizeof("version"), \ + pStorage->version, strlen(pStorage->version), 1); + + add_assoc_stringl_ex(server_info_array, \ + "src_storage_id", sizeof("src_storage_id"), \ + pStorage->src_id, strlen(pStorage->src_id), 1); + + add_assoc_bool_ex(server_info_array, \ + "if_trunk_server", sizeof("if_trunk_server"), \ + pStorage->if_trunk_server); + + add_assoc_long_ex(server_info_array, \ + "upload_priority", sizeof("upload_priority"), \ + pStorage->upload_priority); + + add_assoc_long_ex(server_info_array, \ + "store_path_count", sizeof("store_path_count"),\ + pStorage->store_path_count); + + add_assoc_long_ex(server_info_array, \ + "subdir_count_per_path", \ + sizeof("subdir_count_per_path"), \ + pStorage->subdir_count_per_path); + + add_assoc_long_ex(server_info_array, \ + "storage_port", sizeof("storage_port"), \ + pStorage->storage_port); + + add_assoc_long_ex(server_info_array, \ + "storage_http_port", \ + sizeof("storage_http_port"), \ + pStorage->storage_http_port); + + add_assoc_long_ex(server_info_array, \ + "current_write_path", \ + sizeof("current_write_path"), \ + pStorage->current_write_path); + + add_assoc_long_ex(server_info_array, "status", \ + sizeof("status"), pStorage->status); + add_assoc_long_ex(server_info_array, "total_space", \ + sizeof("total_space"), pStorage->total_mb); + add_assoc_long_ex(server_info_array, "free_space", \ + sizeof("free_space"), pStorage->free_mb); + + pStorageStat = &(pStorage->stat); + + add_assoc_long_ex(server_info_array, \ + "total_upload_count", \ + sizeof("total_upload_count"), \ + pStorageStat->total_upload_count); + + add_assoc_long_ex(server_info_array, \ + "success_upload_count", \ + sizeof("success_upload_count"), \ + pStorageStat->success_upload_count); + + add_assoc_long_ex(server_info_array, \ + "total_append_count", \ + sizeof("total_append_count"), \ + pStorageStat->total_append_count); + + add_assoc_long_ex(server_info_array, \ + "success_append_count", \ + sizeof("success_append_count"), \ + pStorageStat->success_append_count); + + add_assoc_long_ex(server_info_array, \ + "total_modify_count", \ + sizeof("total_modify_count"), \ + pStorageStat->total_modify_count); + + add_assoc_long_ex(server_info_array, \ + "success_modify_count", \ + sizeof("success_modify_count"), \ + pStorageStat->success_modify_count); + + add_assoc_long_ex(server_info_array, \ + "total_truncate_count", \ + sizeof("total_truncate_count"), \ + pStorageStat->total_truncate_count); + + add_assoc_long_ex(server_info_array, \ + "success_truncate_count", \ + sizeof("success_truncate_count"), \ + pStorageStat->success_truncate_count); + + add_assoc_long_ex(server_info_array, \ + "total_set_meta_count", \ + sizeof("total_set_meta_count"), \ + pStorageStat->total_set_meta_count); + + add_assoc_long_ex(server_info_array, \ + "success_set_meta_count", \ + sizeof("success_set_meta_count"), \ + pStorageStat->success_set_meta_count); + + add_assoc_long_ex(server_info_array, \ + "total_delete_count", \ + sizeof("total_delete_count"), \ + pStorageStat->total_delete_count); + + add_assoc_long_ex(server_info_array, \ + "success_delete_count", \ + sizeof("success_delete_count"), \ + pStorageStat->success_delete_count); + + add_assoc_long_ex(server_info_array, \ + "total_download_count", \ + sizeof("total_download_count"), \ + pStorageStat->total_download_count); + + add_assoc_long_ex(server_info_array, \ + "success_download_count", \ + sizeof("success_download_count"), \ + pStorageStat->success_download_count); + + add_assoc_long_ex(server_info_array, \ + "total_get_meta_count", \ + sizeof("total_get_meta_count"), \ + pStorageStat->total_get_meta_count); + + add_assoc_long_ex(server_info_array, \ + "success_get_meta_count", \ + sizeof("success_get_meta_count"), \ + pStorageStat->success_get_meta_count); + + add_assoc_long_ex(server_info_array, \ + "total_create_link_count", \ + sizeof("total_create_link_count"), \ + pStorageStat->total_create_link_count); + + add_assoc_long_ex(server_info_array, \ + "success_create_link_count", \ + sizeof("success_create_link_count"), \ + pStorageStat->success_create_link_count); + + add_assoc_long_ex(server_info_array, \ + "total_delete_link_count", \ + sizeof("total_delete_link_count"), \ + pStorageStat->total_delete_link_count); + + add_assoc_long_ex(server_info_array, \ + "success_delete_link_count", \ + sizeof("success_delete_link_count"), \ + pStorageStat->success_delete_link_count); + add_assoc_long_ex(server_info_array, \ + "total_upload_bytes", \ + sizeof("total_upload_bytes"), \ + pStorageStat->total_upload_bytes); + add_assoc_long_ex(server_info_array, \ + "success_upload_bytes", \ + sizeof("success_upload_bytes"), \ + pStorageStat->success_upload_bytes); + add_assoc_long_ex(server_info_array, \ + "total_append_bytes", \ + sizeof("total_append_bytes"), \ + pStorageStat->total_append_bytes); + add_assoc_long_ex(server_info_array, \ + "success_append_bytes", \ + sizeof("success_append_bytes"), \ + pStorageStat->success_append_bytes); + add_assoc_long_ex(server_info_array, \ + "total_modify_bytes", \ + sizeof("total_modify_bytes"), \ + pStorageStat->total_modify_bytes); + add_assoc_long_ex(server_info_array, \ + "success_modify_bytes", \ + sizeof("success_modify_bytes"), \ + pStorageStat->success_modify_bytes); + add_assoc_long_ex(server_info_array, \ + "total_download_bytes", \ + sizeof("total_download_bytes"), \ + pStorageStat->total_download_bytes); + add_assoc_long_ex(server_info_array, \ + "success_download_bytes", \ + sizeof("success_download_bytes"), \ + pStorageStat->success_download_bytes); + add_assoc_long_ex(server_info_array, \ + "total_sync_in_bytes", \ + sizeof("total_sync_in_bytes"), \ + pStorageStat->total_sync_in_bytes); + add_assoc_long_ex(server_info_array, \ + "success_sync_in_bytes", \ + sizeof("success_sync_in_bytes"), \ + pStorageStat->success_sync_in_bytes); + add_assoc_long_ex(server_info_array, \ + "total_sync_out_bytes", \ + sizeof("total_sync_out_bytes"), \ + pStorageStat->total_sync_out_bytes); + add_assoc_long_ex(server_info_array, \ + "success_sync_out_bytes", \ + sizeof("success_sync_out_bytes"), \ + pStorageStat->success_sync_out_bytes); + add_assoc_long_ex(server_info_array, \ + "total_file_open_count", \ + sizeof("total_file_open_count"), \ + pStorageStat->total_file_open_count); + add_assoc_long_ex(server_info_array, \ + "success_file_open_count", \ + sizeof("success_file_open_count"), \ + pStorageStat->success_file_open_count); + add_assoc_long_ex(server_info_array, \ + "total_file_read_count", \ + sizeof("total_file_read_count"), \ + pStorageStat->total_file_read_count); + add_assoc_long_ex(server_info_array, \ + "success_file_read_count", \ + sizeof("success_file_read_count"), \ + pStorageStat->success_file_read_count); + add_assoc_long_ex(server_info_array, \ + "total_file_write_count", \ + sizeof("total_file_write_count"), \ + pStorageStat->total_file_write_count); + add_assoc_long_ex(server_info_array, \ + "success_file_write_count", \ + sizeof("success_file_write_count"), \ + pStorageStat->success_file_write_count); + add_assoc_long_ex(server_info_array, \ + "last_heart_beat_time", \ + sizeof("last_heart_beat_time"), \ + pStorageStat->last_heart_beat_time); + + add_assoc_long_ex(server_info_array, \ + "last_source_update", \ + sizeof("last_source_update"), \ + pStorageStat->last_source_update); + add_assoc_long_ex(server_info_array, \ + "last_sync_update", \ + sizeof("last_sync_update"), \ + pStorageStat->last_sync_update); + add_assoc_long_ex(server_info_array, \ + "last_synced_timestamp", \ + sizeof("last_synced_timestamp"), \ + pStorageStat->last_synced_timestamp); + } + } +} + +static void php_fdfs_tracker_query_storage_store_impl( \ + INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *group_name; + int group_nlen; + zval *tracker_obj; + HashTable *tracker_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + int store_path_index; + int saved_tracker_sock; + int result; + + argc = ZEND_NUM_ARGS(); + if (argc > 2) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_query_storage_store parameters " \ + "count: %d > 2", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + group_name = NULL; + group_nlen = 0; + tracker_obj = NULL; + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|sa", \ + &group_name, &group_nlen, &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (group_name != NULL && group_nlen > 0) + { + snprintf(new_group_name, sizeof(new_group_name), "%s", group_name); + result = tracker_query_storage_store_with_group(pTrackerServer,\ + new_group_name, &storage_server, &store_path_index); + } + else + { + *new_group_name = '\0'; + result = tracker_query_storage_store_without_group( \ + pTrackerServer, &storage_server, new_group_name, \ + &store_path_index); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + array_init(return_value); + add_assoc_stringl_ex(return_value, "ip_addr", \ + sizeof("ip_addr"), storage_server.ip_addr, \ + strlen(storage_server.ip_addr), 1); + add_assoc_long_ex(return_value, "port", sizeof("port"), \ + storage_server.port); + add_assoc_long_ex(return_value, "sock", sizeof("sock"), -1); + add_assoc_long_ex(return_value, "store_path_index", \ + sizeof("store_path_index"), \ + store_path_index); +} + +static void php_fdfs_tracker_query_storage_store_list_impl( \ + INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + char *group_name; + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + int group_nlen; + zval *server_info_array; + zval *tracker_obj; + HashTable *tracker_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_servers[FDFS_MAX_SERVERS_EACH_GROUP]; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pServer; + ConnectionInfo *pServerEnd; + int store_path_index; + int saved_tracker_sock; + int result; + int storage_count; + + argc = ZEND_NUM_ARGS(); + if (argc > 2) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_query_storage_store_list parameters " \ + "count: %d > 2", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + group_name = NULL; + group_nlen = 0; + tracker_obj = NULL; + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|sa", \ + &group_name, &group_nlen, &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (group_name != NULL && group_nlen > 0) + { + snprintf(new_group_name, sizeof(new_group_name), "%s", group_name); + result = tracker_query_storage_store_list_with_group(pTrackerServer,\ + new_group_name, storage_servers, FDFS_MAX_SERVERS_EACH_GROUP, \ + &storage_count, &store_path_index); + } + else + { + *new_group_name = '\0'; + result = tracker_query_storage_store_list_without_group( \ + pTrackerServer, storage_servers, \ + FDFS_MAX_SERVERS_EACH_GROUP, &storage_count, \ + new_group_name, &store_path_index); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + array_init(return_value); + + pServerEnd = storage_servers + storage_count; + for (pServer=storage_servers; pServerip_addr, \ + strlen(pServer->ip_addr), 1); + add_assoc_long_ex(server_info_array, "port", sizeof("port"), \ + pServer->port); + add_assoc_long_ex(server_info_array, "sock", sizeof("sock"), -1); + add_assoc_long_ex(server_info_array, "store_path_index", \ + sizeof("store_path_index"), \ + store_path_index); + } +} + +static void php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext, const byte cmd, \ + const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + zval *tracker_obj; + HashTable *tracker_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 2; + } + else + { + min_param_count = 2; + max_param_count = 3; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_do_query_storage parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|a", \ + &file_id, &file_id_len, &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|a", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + result = tracker_do_query_storage(pTrackerServer, &storage_server, \ + cmd, group_name, remote_filename); + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + array_init(return_value); + add_assoc_stringl_ex(return_value, "ip_addr", \ + sizeof("ip_addr"), storage_server.ip_addr, \ + strlen(storage_server.ip_addr), 1); + add_assoc_long_ex(return_value, "port", sizeof("port"), \ + storage_server.port); + add_assoc_long_ex(return_value, "sock", sizeof("sock"), -1); +} + +static void php_fdfs_tracker_delete_storage_impl( \ + INTERNAL_FUNCTION_PARAMETERS, + FDFSPhpContext *pContext) +{ + int argc; + int group_name_len; + int storage_ip_len; + char *group_name; + char *storage_ip; + + argc = ZEND_NUM_ARGS(); + if (argc != 2) + { + logError("file: "__FILE__", line: %d, " \ + "tracker_delete_storage parameters " \ + "count: %d != 2", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss", \ + &group_name, &group_name_len, &storage_ip, &storage_ip_len) + == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (group_name_len == 0 || storage_ip_len == 0) + { + logError("file: "__FILE__", line: %d, " \ + "group name length: %d or storage ip length: %d = 0!",\ + __LINE__, group_name_len, storage_ip_len); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + pContext->err_no = tracker_delete_storage(pContext->pTrackerGroup, \ + group_name, storage_ip); + if (pContext->err_no == 0) + { + RETURN_BOOL(true); + } + else + { + RETURN_BOOL(false); + } +} + +static void php_fdfs_storage_delete_file_impl( \ + INTERNAL_FUNCTION_PARAMETERS, + FDFSPhpContext *pContext, const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 3; + } + else + { + min_param_count = 2; + max_param_count = 4; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_delete_file parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|aa", \ + &file_id, &file_id_len, &tracker_obj, &storage_obj) \ + == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|aa", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + result = storage_delete_file(pTrackerServer, pStorageServer, \ + group_name, remote_filename); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + RETURN_BOOL(true); +} + +static void php_fdfs_storage_truncate_file_impl( \ + INTERNAL_FUNCTION_PARAMETERS, + FDFSPhpContext *pContext, const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + long truncated_file_size = 0; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 4; + } + else + { + min_param_count = 2; + max_param_count = 5; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_truncate_file parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "s|laa", &file_id, &file_id_len, \ + &truncated_file_size, &tracker_obj, \ + &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "ss|laa", &group_name, &group_nlen, &remote_filename, \ + &filename_len, &truncated_file_size, &tracker_obj, \ + &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + result = storage_truncate_file(pTrackerServer, pStorageServer, \ + group_name, remote_filename, truncated_file_size); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + RETURN_BOOL(true); +} + +static void php_fdfs_storage_download_file_to_callback_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + zval *download_callback; + int group_nlen; + int filename_len; + long file_offset; + long download_bytes; + int64_t file_size; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + HashTable *callback_hash; + php_fdfs_callback_t php_callback; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 2; + max_param_count = 6; + } + else + { + min_param_count = 3; + max_param_count = 7; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_download_file_to_buff parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + file_offset = 0; + download_bytes = 0; + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "sa|llaa", &file_id, &file_id_len, \ + &download_callback, &file_offset, &download_bytes, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ssa|llaa", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &download_callback, &file_offset, &download_bytes, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + callback_hash = Z_ARRVAL_P(download_callback); + result = php_fdfs_get_callback_from_hash(callback_hash, \ + &php_callback); + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + pContext->err_no = result; + RETURN_BOOL(false); + } + + result = storage_download_file_ex(pTrackerServer, pStorageServer, \ + group_name, remote_filename, file_offset, download_bytes, \ + php_fdfs_download_callback, (void *)&php_callback, &file_size); + if (tracker_hash != NULL && pTrackerServer->sock != saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + pContext->err_no = result; + RETURN_BOOL(false); + } + RETURN_BOOL(true); +} + +static void php_fdfs_storage_download_file_to_buff_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + char *file_buff; + char *new_file_buff; + int group_nlen; + int filename_len; + long file_offset; + long download_bytes; + int64_t file_size; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 5; + } + else + { + min_param_count = 2; + max_param_count = 6; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_download_file_to_buff parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + file_offset = 0; + download_bytes = 0; + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|llaa", \ + &file_id, &file_id_len, &file_offset, &download_bytes, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|llaa", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &file_offset, &download_bytes, &tracker_obj, &storage_obj) \ + == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + result=storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_BUFF, group_name, remote_filename, \ + file_offset, download_bytes, &file_buff, NULL, &file_size); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + pContext->err_no = result; + RETURN_BOOL(false); + } + + new_file_buff = (char *)emalloc(file_size + 1); + if (new_file_buff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "emalloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)file_size + 1, errno, STRERROR(errno)); + free(file_buff); + pContext->err_no = errno != 0 ? errno : ENOMEM; + RETURN_BOOL(false); + } + + memcpy(new_file_buff, file_buff, file_size); + *(new_file_buff + file_size) = '\0'; + free(file_buff); + + pContext->err_no = 0; + RETURN_STRINGL(new_file_buff, file_size, 0); +} + +static void php_fdfs_storage_download_file_to_file_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + char *local_filename; + int group_nlen; + int remote_file_nlen; + int local_file_nlen; + long file_offset; + long download_bytes; + int64_t file_size; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 2; + max_param_count = 6; + } + else + { + min_param_count = 3; + max_param_count = 7; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_set_metadata parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + file_offset = 0; + download_bytes = 0; + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|llaa",\ + &file_id, &file_id_len, &local_filename, \ + &local_file_nlen, &file_offset, &download_bytes, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sss|llaa", \ + &group_name, &group_nlen, &remote_filename, &remote_file_nlen,\ + &local_filename, &local_file_nlen, &file_offset, \ + &download_bytes, &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + result=storage_do_download_file_ex(pTrackerServer, pStorageServer, \ + FDFS_DOWNLOAD_TO_FILE, group_name, remote_filename, \ + file_offset, download_bytes, &local_filename, NULL, &file_size); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + else + { + RETURN_BOOL(true); + } +} + +static void php_fdfs_storage_get_metadata_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + FDFSMetaData *meta_list; + FDFSMetaData *pMetaData; + FDFSMetaData *pMetaEnd; + int meta_count; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 3; + } + else + { + min_param_count = 2; + max_param_count = 4; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_get_metadata parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|aa", \ + &file_id, &file_id_len, &tracker_obj, &storage_obj) \ + == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|aa", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + result = storage_get_metadata(pTrackerServer, pStorageServer, \ + group_name, remote_filename, &meta_list, &meta_count); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + array_init(return_value); + if (meta_list != NULL) + { + pMetaEnd = meta_list + meta_count; + for (pMetaData=meta_list; pMetaDataname, \ + strlen(pMetaData->name)+1, pMetaData->value,\ + strlen(pMetaData->value), 1); + } + + free(meta_list); + } +} + +static void php_fdfs_storage_file_exist_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + int result; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 3; + } + else + { + min_param_count = 2; + max_param_count = 4; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_file_exist parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|aa", \ + &file_id, &file_id_len, &tracker_obj, &storage_obj) \ + == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|aa", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + result = storage_file_exist(pTrackerServer, pStorageServer, \ + group_name, remote_filename); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result == 0) + { + RETURN_BOOL(true); + } + else + { + RETURN_BOOL(false); + } +} + +static void php_fdfs_tracker_query_storage_list_impl( \ + INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext, const bool bFileId) +{ + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + zval *tracker_obj; + zval *server_info_array; + HashTable *tracker_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_servers[FDFS_MAX_SERVERS_EACH_GROUP]; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pServer; + ConnectionInfo *pServerEnd; + int result; + int server_count; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 2; + } + else + { + min_param_count = 2; + max_param_count = 3; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "fastdfs_tracker_query_storage_list parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|a", \ + &file_id, &file_id_len, &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|a", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &tracker_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + result = tracker_query_storage_list(pTrackerServer, storage_servers, \ + FDFS_MAX_SERVERS_EACH_GROUP, &server_count, \ + group_name, remote_filename); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + pContext->err_no = result; + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + array_init(return_value); + + pServerEnd = storage_servers + server_count; + for (pServer=storage_servers; pServerip_addr, \ + strlen(pServer->ip_addr), 1); + add_assoc_long_ex(server_info_array, "port", sizeof("port"), \ + pServer->port); + add_assoc_long_ex(server_info_array,"sock",sizeof("sock"),-1); + } +} + +static int php_fdfs_upload_callback(void *arg, const int64_t file_size, int sock) +{ + php_fdfs_upload_callback_t *pUploadCallback; + zval *args[2]; + zval zsock; + zval ret; + zval null_args; + int result; + TSRMLS_FETCH(); + + ZVAL_NULL(&ret); + ZVAL_LONG(&zsock, sock); + + pUploadCallback = (php_fdfs_upload_callback_t *)arg; + if (pUploadCallback->callback.args == NULL) + { + ZVAL_NULL(&null_args); + pUploadCallback->callback.args = &null_args; + } + args[0] = &zsock; + args[1] = pUploadCallback->callback.args; + + if (call_user_function(EG(function_table), NULL, \ + pUploadCallback->callback.func_name, + &ret, 2, args TSRMLS_CC) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "call callback function: %s fail", __LINE__, \ + Z_STRVAL_P(pUploadCallback->callback.func_name)); + return EINVAL; + } + + if (ret.type == IS_LONG || ret.type == IS_BOOL) + { + result = ret.value.lval == 0 ? EFAULT : 0; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "callback function return invalid value type: %d", \ + __LINE__, ret.type); + result = EINVAL; + } + + return result; +} + +static int php_fdfs_download_callback(void *arg, const int64_t file_size, \ + const char *data, const int current_size) +{ + php_fdfs_callback_t *pCallback; + zval *args[3]; + zval zfilesize; + zval zdata; + zval ret; + zval null_args; + int result; + TSRMLS_FETCH(); + + ZVAL_NULL(&ret); + ZVAL_LONG(&zfilesize, file_size); + ZVAL_STRINGL(&zdata, (char *)data, current_size, 0); + + pCallback = (php_fdfs_callback_t *)arg; + if (pCallback->args == NULL) + { + ZVAL_NULL(&null_args); + pCallback->args = &null_args; + } + args[0] = pCallback->args; + args[1] = &zfilesize; + args[2] = &zdata; + if (call_user_function(EG(function_table), NULL, \ + pCallback->func_name, + &ret, 3, args TSRMLS_CC) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "call callback function: %s fail", __LINE__, \ + Z_STRVAL_P(pCallback->func_name)); + return EINVAL; + } + + if (ret.type == IS_LONG || ret.type == IS_BOOL) + { + result = ret.value.lval == 0 ? EFAULT : 0; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "callback function return invalid value type: %d", \ + __LINE__, ret.type); + result = EINVAL; + } + + return result; +} + +/* +string/array fastdfs_storage_upload_by_filename(string local_filename + [, string file_ext_name, array meta_list, string group_name, + array tracker_server, array storage_server]) +return string/array for success, false for error +*/ +static void php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext, const byte cmd, \ + const byte upload_type, const bool bFileId) +{ + int result; + int argc; + char *local_filename; + int filename_len; + char *file_ext_name; + zval *callback_obj; + zval *ext_name_obj; + zval *metadata_obj; + zval *tracker_obj; + zval *storage_obj; + zval *group_name_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + FDFSMetaData *meta_list; + int meta_count; + int store_path_index; + int saved_tracker_sock; + int saved_storage_sock; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + + argc = ZEND_NUM_ARGS(); + if (argc < 1 || argc > 6) + { + logError("file: "__FILE__", line: %d, " \ + "storage_upload_file parameters " \ + "count: %d < 1 or > 6", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + local_filename = NULL; + filename_len = 0; + callback_obj = NULL; + ext_name_obj = NULL; + metadata_obj = NULL; + group_name_obj = NULL; + tracker_obj = NULL; + storage_obj = NULL; + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "a|zazaa", &callback_obj, &ext_name_obj, \ + &metadata_obj, &group_name_obj, &tracker_obj, \ + &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "s|zazaa", &local_filename, &filename_len, \ + &ext_name_obj, &metadata_obj, &group_name_obj, \ + &tracker_obj, &storage_obj); + } + + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (ext_name_obj == NULL) + { + file_ext_name = NULL; + } + else + { + if (ext_name_obj->type == IS_NULL) + { + file_ext_name = NULL; + } + else if (ext_name_obj->type == IS_STRING) + { + file_ext_name = ext_name_obj->value.str.val; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "file_ext_name is not a string, type=%d!", \ + __LINE__, ext_name_obj->type); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + if (group_name_obj != NULL && group_name_obj->type == IS_STRING) + { + snprintf(group_name, sizeof(group_name), "%s", \ + group_name_obj->value.str.val); + } + else + { + *group_name = '\0'; + } + *remote_filename = '\0'; + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + store_path_index = 0; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + zval **data; + zval ***ppp; + + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + data = NULL; + ppp = &data; + if (zend_hash_find(storage_hash, "store_path_index", \ + sizeof("store_path_index"), (void **)ppp) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "key \"store_path_index\" not exist!", \ + __LINE__); + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + + if ((*data)->type == IS_LONG) + { + store_path_index = (*data)->value.lval; + } + else if ((*data)->type == IS_STRING) + { + store_path_index = atoi(Z_STRVAL_PP(data)); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "key \"store_path_index\" is invalid, " \ + "type=%d!", __LINE__, (*data)->type); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + if (metadata_obj == NULL) + { + meta_list = NULL; + meta_count = 0; + } + else + { + result = fastdfs_convert_metadata_to_array(metadata_obj, \ + &meta_list, &meta_count); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + } + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_upload_by_filename_ex(pTrackerServer, pStorageServer, \ + store_path_index, cmd, local_filename, file_ext_name, \ + meta_list, meta_count, group_name, remote_filename); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + char *buff; + int buff_len; + + buff = local_filename; + buff_len = filename_len; + + result = storage_do_upload_file(pTrackerServer, pStorageServer, \ + store_path_index, cmd, FDFS_UPLOAD_BY_BUFF, buff, NULL, \ + buff_len, NULL, NULL, file_ext_name, meta_list, meta_count, \ + group_name, remote_filename); + } + else //by callback + { + HashTable *callback_hash; + php_fdfs_upload_callback_t php_callback; + + callback_hash = Z_ARRVAL_P(callback_obj); + result = php_fdfs_get_upload_callback_from_hash(callback_hash, \ + &php_callback); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + result = storage_upload_by_callback_ex(pTrackerServer, \ + pStorageServer, store_path_index, cmd, \ + php_fdfs_upload_callback, (void *)&php_callback, \ + php_callback.file_size, file_ext_name, meta_list, \ + meta_count, group_name, remote_filename); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (meta_list != NULL) + { + free(meta_list); + } + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + if (bFileId) + { + char file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + int file_id_len; + + file_id_len = sprintf(file_id, "%s%c%s", group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + RETURN_STRINGL(file_id, file_id_len, 1); + } + else + { + array_init(return_value); + + add_assoc_stringl_ex(return_value, "group_name", \ + sizeof("group_name"), group_name, \ + strlen(group_name), 1); + add_assoc_stringl_ex(return_value, "filename", \ + sizeof("filename"), remote_filename, \ + strlen(remote_filename), 1); + } +} + +static void php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const byte upload_type, const bool bFileId) +{ + int result; + int argc; + zval *callback_obj; + char *local_filename; + char *master_filename; + char *prefix_name; + char *file_ext_name; + zval *ext_name_obj; + zval *metadata_obj; + zval *tracker_obj; + zval *storage_obj; + char *group_name; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + FDFSMetaData *meta_list; + int meta_count; + int filename_len; + int group_name_len; + int master_filename_len; + int prefix_name_len; + int saved_tracker_sock; + int saved_storage_sock; + int min_param_count; + int max_param_count; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char remote_filename[128]; + + if (bFileId) + { + min_param_count = 3; + max_param_count = 7; + } + else + { + min_param_count = 4; + max_param_count = 8; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_upload_slave_file parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + local_filename = NULL; + filename_len = 0; + callback_obj = NULL; + ext_name_obj = NULL; + metadata_obj = NULL; + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *master_file_id; + int master_file_id_len; + + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "ass|zaaa", &callback_obj, \ + &master_file_id, &master_file_id_len, \ + &prefix_name, &prefix_name_len, &ext_name_obj, \ + &metadata_obj, &tracker_obj, &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "sss|zaaa", &local_filename, &filename_len, \ + &master_file_id, &master_file_id_len, \ + &prefix_name, &prefix_name_len, &ext_name_obj, \ + &metadata_obj, &tracker_obj, &storage_obj); + } + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", master_file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "master_file_id is invalid, master_file_id=%s",\ + __LINE__, master_file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + master_filename = pSeperator + 1; + } + else + { + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, + "asss|zaaa", &callback_obj, \ + &group_name, &group_name_len, \ + &master_filename, &master_filename_len, \ + &prefix_name, &prefix_name_len, &ext_name_obj,\ + &metadata_obj, &tracker_obj, &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, + "ssss|zaaa", &local_filename, &filename_len, \ + &group_name, &group_name_len, \ + &master_filename, &master_filename_len, \ + &prefix_name, &prefix_name_len, &ext_name_obj,\ + &metadata_obj, &tracker_obj, &storage_obj); + } + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + if (ext_name_obj == NULL) + { + file_ext_name = NULL; + } + else + { + if (ext_name_obj->type == IS_NULL) + { + file_ext_name = NULL; + } + else if (ext_name_obj->type == IS_STRING) + { + file_ext_name = ext_name_obj->value.str.val; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "file_ext_name is not a string, type=%d!", \ + __LINE__, ext_name_obj->type); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + *remote_filename = '\0'; + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + saved_storage_sock = pStorageServer->sock; + } + + if (metadata_obj == NULL) + { + meta_list = NULL; + meta_count = 0; + } + else + { + result = fastdfs_convert_metadata_to_array(metadata_obj, \ + &meta_list, &meta_count); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + } + + snprintf(new_group_name, sizeof(new_group_name), "%s", group_name); + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_upload_slave_by_filename(pTrackerServer, \ + pStorageServer, local_filename, master_filename, \ + prefix_name, file_ext_name, meta_list, meta_count, \ + new_group_name, remote_filename); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + result = storage_upload_slave_by_filebuff(pTrackerServer, \ + pStorageServer, local_filename, filename_len, \ + master_filename, prefix_name, file_ext_name, \ + meta_list, meta_count, new_group_name, remote_filename); + } + else //by callback + { + HashTable *callback_hash; + php_fdfs_upload_callback_t php_callback; + + callback_hash = Z_ARRVAL_P(callback_obj); + result = php_fdfs_get_upload_callback_from_hash(callback_hash, \ + &php_callback); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + result = storage_upload_slave_by_callback(pTrackerServer, \ + pStorageServer, php_fdfs_upload_callback, \ + (void *)&php_callback, php_callback.file_size, \ + master_filename, prefix_name, file_ext_name, meta_list,\ + meta_count, new_group_name, remote_filename); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (meta_list != NULL) + { + free(meta_list); + } + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + + if (bFileId) + { + char file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + int file_id_len; + + file_id_len = sprintf(file_id, "%s%c%s", new_group_name, \ + FDFS_FILE_ID_SEPERATOR, remote_filename); + RETURN_STRINGL(file_id, file_id_len, 1); + } + else + { + array_init(return_value); + + add_assoc_stringl_ex(return_value, "group_name", \ + sizeof("group_name"), new_group_name, \ + strlen(new_group_name), 1); + add_assoc_stringl_ex(return_value, "filename", \ + sizeof("filename"), remote_filename, \ + strlen(remote_filename), 1); + } +} + +/* +boolean fastdfs_storage_append_by_filename(string local_filename, + string group_name, appender_filename, + [array tracker_server, array storage_server]) +return true for success, false for error +*/ +static void php_fdfs_storage_append_file_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const byte upload_type, const bool bFileId) +{ + int result; + int argc; + zval *callback_obj; + char *local_filename; + char *appender_filename; + zval *tracker_obj; + zval *storage_obj; + char *group_name; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + int filename_len; + int group_name_len; + int appender_filename_len; + int saved_tracker_sock; + int saved_storage_sock; + int min_param_count; + int max_param_count; + + if (bFileId) + { + min_param_count = 2; + max_param_count = 4; + } + else + { + min_param_count = 3; + max_param_count = 5; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_append_file parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + local_filename = NULL; + filename_len = 0; + callback_obj = NULL; + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *appender_file_id; + int appender_file_id_len; + + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "as|aa", &callback_obj, &appender_file_id, \ + &appender_file_id_len, &tracker_obj, &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "ss|aa", &local_filename, &filename_len, \ + &appender_file_id, &appender_file_id_len, \ + &tracker_obj, &storage_obj); + } + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", appender_file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "appender_file_id is invalid, " \ + "appender_file_id=%s", \ + __LINE__, appender_file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + appender_filename = pSeperator + 1; + } + else + { + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "ass|aa", &callback_obj, &group_name, &group_name_len, \ + &appender_filename, &appender_filename_len, \ + &tracker_obj, &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "sss|aa", &local_filename, &filename_len, \ + &group_name, &group_name_len, \ + &appender_filename, &appender_filename_len, \ + &tracker_obj, &storage_obj); + } + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + saved_storage_sock = pStorageServer->sock; + } + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_append_by_filename(pTrackerServer, \ + pStorageServer, local_filename, group_name, \ + appender_filename); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + result = storage_append_by_filebuff(pTrackerServer, \ + pStorageServer, local_filename, filename_len, \ + group_name, appender_filename); + } + else + { + HashTable *callback_hash; + php_fdfs_upload_callback_t php_callback; + + callback_hash = Z_ARRVAL_P(callback_obj); + result = php_fdfs_get_upload_callback_from_hash(callback_hash, \ + &php_callback); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + result = storage_append_by_callback(pTrackerServer, \ + pStorageServer, php_fdfs_upload_callback, \ + (void *)&php_callback, php_callback.file_size, \ + group_name, appender_filename); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result == 0) + { + RETURN_BOOL(true); + } + else + { + RETURN_BOOL(false); + } +} + +/* +boolean fastdfs_storage_modify_by_filename(string local_filename, + long file_offset, string group_name, appender_filename, + [array tracker_server, array storage_server]) +return true for success, false for error +*/ +static void php_fdfs_storage_modify_file_impl( \ + INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext, \ + const byte upload_type, const bool bFileId) +{ + int result; + int argc; + zval *callback_obj; + char *local_filename; + char *appender_filename; + zval *tracker_obj; + zval *storage_obj; + char *group_name; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + int filename_len; + int group_name_len; + int appender_filename_len; + int saved_tracker_sock; + int saved_storage_sock; + int min_param_count; + int max_param_count; + long file_offset = 0; + + if (bFileId) + { + min_param_count = 3; + max_param_count = 5; + } + else + { + min_param_count = 4; + max_param_count = 6; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_modify_file parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + local_filename = NULL; + filename_len = 0; + callback_obj = NULL; + tracker_obj = NULL; + storage_obj = NULL; + if (bFileId) + { + char *pSeperator; + char *appender_file_id; + int appender_file_id_len; + + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "als|aa", &callback_obj, &file_offset, \ + &appender_file_id, &appender_file_id_len, \ + &tracker_obj, &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "sls|aa", &local_filename, &filename_len, \ + &file_offset, &appender_file_id, &appender_file_id_len, \ + &tracker_obj, &storage_obj); + } + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", appender_file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "appender_file_id is invalid, " \ + "appender_file_id=%s", \ + __LINE__, appender_file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + appender_filename = pSeperator + 1; + } + else + { + if (upload_type == FDFS_UPLOAD_BY_CALLBACK) + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "alss|aa", &callback_obj, &file_offset, \ + &group_name, &group_name_len, \ + &appender_filename, &appender_filename_len, \ + &tracker_obj, &storage_obj); + } + else + { + result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, \ + "slss|aa", &local_filename, &filename_len, \ + &file_offset, &group_name, &group_name_len, \ + &appender_filename, &appender_filename_len, \ + &tracker_obj, &storage_obj); + } + if (result == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + saved_storage_sock = pStorageServer->sock; + } + + if (upload_type == FDFS_UPLOAD_BY_FILE) + { + result = storage_modify_by_filename(pTrackerServer, \ + pStorageServer, local_filename, file_offset, \ + group_name, appender_filename); + } + else if (upload_type == FDFS_UPLOAD_BY_BUFF) + { + result = storage_modify_by_filebuff(pTrackerServer, \ + pStorageServer, local_filename, \ + file_offset, filename_len, \ + group_name, appender_filename); + } + else + { + HashTable *callback_hash; + php_fdfs_upload_callback_t php_callback; + + callback_hash = Z_ARRVAL_P(callback_obj); + result = php_fdfs_get_upload_callback_from_hash( \ + callback_hash, &php_callback); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + + result = storage_modify_by_callback(pTrackerServer, \ + pStorageServer, php_fdfs_upload_callback, \ + (void *)&php_callback, file_offset, \ + php_callback.file_size, group_name, \ + appender_filename); + } + + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (result == 0) + { + RETURN_BOOL(true); + } + else + { + RETURN_BOOL(false); + } +} + +static void php_fdfs_storage_set_metadata_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext, const bool bFileId) +{ + int result; + int argc; + char *group_name; + char *remote_filename; + char *op_type_str; + char op_type; + int group_nlen; + int filename_len; + int op_type_len; + zval *metadata_obj; + zval *tracker_obj; + zval *storage_obj; + HashTable *tracker_hash; + HashTable *storage_hash; + ConnectionInfo tracker_server; + ConnectionInfo storage_server; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + FDFSMetaData *meta_list; + int meta_count; + int min_param_count; + int max_param_count; + int saved_tracker_sock; + int saved_storage_sock; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + min_param_count = 1; + max_param_count = 5; + } + else + { + min_param_count = 2; + max_param_count = 6; + } + + argc = ZEND_NUM_ARGS(); + if (argc < min_param_count || argc > max_param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_set_metadata parameters " \ + "count: %d < %d or > %d", __LINE__, argc, \ + min_param_count, max_param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + tracker_obj = NULL; + storage_obj = NULL; + op_type_str = NULL; + op_type_len = 0; + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sa|saa", \ + &file_id, &file_id_len, &metadata_obj, &op_type_str, \ + &op_type_len, &tracker_obj, &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ssa|saa", \ + &group_name, &group_nlen, &remote_filename, &filename_len, \ + &metadata_obj, &op_type_str, &op_type_len, &tracker_obj, \ + &storage_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (tracker_obj == NULL) + { + pTrackerServer = tracker_get_connection_no_pool( \ + pContext->pTrackerGroup); + if (pTrackerServer == NULL) + { + pContext->err_no = ENOENT; + RETURN_BOOL(false); + } + saved_tracker_sock = -1; + tracker_hash = NULL; + } + else + { + pTrackerServer = &tracker_server; + tracker_hash = Z_ARRVAL_P(tracker_obj); + if ((result=php_fdfs_get_server_from_hash(tracker_hash, \ + pTrackerServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_tracker_sock = pTrackerServer->sock; + } + + if (storage_obj == NULL) + { + pStorageServer = NULL; + storage_hash = NULL; + saved_storage_sock = -1; + } + else + { + pStorageServer = &storage_server; + storage_hash = Z_ARRVAL_P(storage_obj); + if ((result=php_fdfs_get_server_from_hash(storage_hash, \ + pStorageServer)) != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + saved_storage_sock = pStorageServer->sock; + } + + if (metadata_obj == NULL) + { + meta_list = NULL; + meta_count = 0; + } + else + { + result = fastdfs_convert_metadata_to_array(metadata_obj, \ + &meta_list, &meta_count); + if (result != 0) + { + pContext->err_no = result; + RETURN_BOOL(false); + } + } + + if (op_type_str == NULL) + { + op_type = STORAGE_SET_METADATA_FLAG_MERGE; + } + else if (TO_UPPERCASE(*op_type_str) == STORAGE_SET_METADATA_FLAG_MERGE) + { + op_type = STORAGE_SET_METADATA_FLAG_MERGE; + } + else if (TO_UPPERCASE(*op_type_str) == STORAGE_SET_METADATA_FLAG_OVERWRITE) + { + op_type = STORAGE_SET_METADATA_FLAG_OVERWRITE; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "invalid op_type: %s!", __LINE__, op_type_str); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + result = storage_set_metadata(pTrackerServer, pStorageServer, \ + group_name, remote_filename, \ + meta_list, meta_count, op_type); + if (tracker_hash != NULL && pTrackerServer->sock != \ + saved_tracker_sock) + { + CLEAR_HASH_SOCK_FIELD(tracker_hash) + } + if (pStorageServer != NULL && pStorageServer->sock != \ + saved_storage_sock) + { + CLEAR_HASH_SOCK_FIELD(storage_hash) + } + + pContext->err_no = result; + if (meta_list != NULL) + { + free(meta_list); + } + if (result != 0) + { + if (tracker_obj == NULL) + { + conn_pool_disconnect_server(pTrackerServer); + } + RETURN_BOOL(false); + } + else + { + RETURN_BOOL(true); + } +} + +static void php_fdfs_http_gen_token_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int result; + int argc; + char *file_id; + int file_id_len; + long ts; + char token[64]; + + argc = ZEND_NUM_ARGS(); + if (argc != 2) + { + logError("file: "__FILE__", line: %d, " \ + "storage_upload_file parameters " \ + "count: %d != 2", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl", \ + &file_id, &file_id_len, &ts) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + result = fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \ + (int)ts, token); + pContext->err_no = result; + if (result != 0) + { + RETURN_BOOL(false); + } + + RETURN_STRINGL(token, strlen(token), 1); +} + +static void php_fdfs_send_data_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int argc; + long sock; + char *buff; + int buff_len; + + argc = ZEND_NUM_ARGS(); + if (argc != 2) + { + logError("file: "__FILE__", line: %d, " \ + "send_data parameters " \ + "count: %d != 2", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ls", \ + &sock, &buff, &buff_len) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if ((pContext->err_no=tcpsenddata_nb(sock, buff, \ + buff_len, g_fdfs_network_timeout)) != 0) + { + RETURN_BOOL(false); + } + + RETURN_BOOL(true); +} + +static void php_fdfs_get_file_info_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext, const bool bFileId) +{ + int result; + int argc; + char *group_name; + char *remote_filename; + int group_nlen; + int filename_len; + int param_count; + FDFSFileInfo file_info; + char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; + + if (bFileId) + { + param_count = 1; + } + else + { + param_count = 2; + } + + argc = ZEND_NUM_ARGS(); + if (argc != param_count) + { + logError("file: "__FILE__", line: %d, " \ + "storage_upload_file parameters " \ + "count: %d != %d", __LINE__, argc, param_count); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (bFileId) + { + char *pSeperator; + char *file_id; + int file_id_len; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s", \ + &file_id, &file_id_len) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); + pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "file_id is invalid, file_id=%s", \ + __LINE__, file_id); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + *pSeperator = '\0'; + group_name = new_file_id; + remote_filename = pSeperator + 1; + } + else + { + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss", \ + &group_name, &group_nlen, &remote_filename, \ + &filename_len) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + result = fdfs_get_file_info_ex(group_name, remote_filename, true, &file_info); + pContext->err_no = result; + if (result != 0) + { + RETURN_BOOL(false); + } + + array_init(return_value); + add_assoc_long_ex(return_value, "source_id", \ + sizeof("source_id"), file_info.source_id); + add_assoc_long_ex(return_value, "create_timestamp", \ + sizeof("create_timestamp"), file_info.create_timestamp); + add_assoc_long_ex(return_value, "file_size", \ + sizeof("file_size"), (long)file_info.file_size); + add_assoc_stringl_ex(return_value, "source_ip_addr", \ + sizeof("source_ip_addr"), file_info.source_ip_addr, \ + strlen(file_info.source_ip_addr), 1); + add_assoc_long_ex(return_value, "crc32", \ + sizeof("crc32"), file_info.crc32); +} + +/* +string fastdfs_gen_slave_filename(string master_filename, string prefix_name + [, string file_ext_name]) +return slave filename string for success, false for error +*/ +static void php_fdfs_gen_slave_filename_impl(INTERNAL_FUNCTION_PARAMETERS, \ + FDFSPhpContext *pContext) +{ + int result; + int argc; + char *master_filename; + int master_filename_len; + char *prefix_name; + int prefix_name_len; + int filename_len; + zval *ext_name_obj; + char *file_ext_name; + int file_ext_name_len; + char filename[128]; + + argc = ZEND_NUM_ARGS(); + if (argc != 2 && argc != 3) + { + logError("file: "__FILE__", line: %d, " \ + "storage_upload_file parameters " \ + "count: %d != 2 or 3", __LINE__, argc); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + ext_name_obj = NULL; + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|z", \ + &master_filename, &master_filename_len, &prefix_name, \ + &prefix_name_len, &ext_name_obj) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + if (ext_name_obj == NULL) + { + file_ext_name = NULL; + file_ext_name_len = 0; + } + else + { + if (ext_name_obj->type == IS_NULL) + { + file_ext_name = NULL; + file_ext_name_len = 0; + } + else if (ext_name_obj->type == IS_STRING) + { + file_ext_name = ext_name_obj->value.str.val; + file_ext_name_len = ext_name_obj->value.str.len; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "file_ext_name is not a string, type=%d!", \ + __LINE__, ext_name_obj->type); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + } + + if (master_filename_len + prefix_name_len + file_ext_name_len + 1 \ + >= sizeof(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "filename length is too long!", __LINE__); + pContext->err_no = EINVAL; + RETURN_BOOL(false); + } + + result = fdfs_gen_slave_filename(master_filename, \ + prefix_name, file_ext_name, filename, &filename_len); + pContext->err_no = result; + if (result != 0) + { + RETURN_BOOL(false); + } + + RETURN_STRINGL(filename, filename_len, 1); +} + +/* +array fastdfs_tracker_get_connection() +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_get_connection) +{ + php_fdfs_tracker_get_connection_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context); +} + +/* +boolean fastdfs_tracker_make_all_connections() +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_make_all_connections) +{ + php_fdfs_tracker_make_all_connections_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +/* +boolean fastdfs_tracker_close_all_connections() +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_close_all_connections) +{ + php_fdfs_tracker_close_all_connections_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +/* +array fastdfs_connect_server(string ip_addr, int port) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_connect_server) +{ + php_fdfs_connect_server_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context); +} + +/* +boolean fastdfs_disconnect_server(array serverInfo) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_disconnect_server) +{ + php_fdfs_disconnect_server_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context); +} + +/* +boolean fastdfs_active_test(array serverInfo) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_active_test) +{ + php_fastdfs_active_test_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context); +} + +/* +long fastdfs_get_last_error_no() +return last error no +*/ +ZEND_FUNCTION(fastdfs_get_last_error_no) +{ + RETURN_LONG(php_context.err_no); +} + +/* +string fastdfs_get_last_error_info() +return last error info +*/ +ZEND_FUNCTION(fastdfs_get_last_error_info) +{ + char *error_info; + + error_info = STRERROR(php_context.err_no); + RETURN_STRINGL(error_info, strlen(error_info), 1); +} + +/* +string fastdfs_client_version() +return client library version +*/ +ZEND_FUNCTION(fastdfs_client_version) +{ + char szVersion[16]; + int len; + + len = sprintf(szVersion, "%d.%02d", \ + g_fdfs_version.major, g_fdfs_version.minor); + + RETURN_STRINGL(szVersion, len, 1); +} + +/* +array fastdfs_tracker_list_groups([string group_name, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_list_groups) +{ + php_fdfs_tracker_list_groups_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context); +} + +/* +array fastdfs_tracker_query_storage_store([string group_name, + array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_store) +{ + php_fdfs_tracker_query_storage_store_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +/* +array fastdfs_tracker_query_storage_store_list([string group_name, + array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_store_list) +{ + php_fdfs_tracker_query_storage_store_list_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +/* +array fastdfs_tracker_query_storage_update(string group_name, + string remote_filename [, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_update) +{ + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE, false); +} + +/* +array fastdfs_tracker_query_storage_fetch(string group_name, + string remote_filename [, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_fetch) +{ + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE, false); +} + +/* +array fastdfs_tracker_query_storage_list(string group_name, + string remote_filename [, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_list) +{ + php_fdfs_tracker_query_storage_list_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, false); +} + +/* +array fastdfs_tracker_query_storage_update1(string file_id, + [, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_update1) +{ + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE, true); +} + +/* +array fastdfs_tracker_query_storage_fetch1(string file_id + [, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_fetch1) +{ + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE, true); +} + +/* +array fastdfs_tracker_query_storage_list1(string file_id + [, array tracker_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_query_storage_list1) +{ + php_fdfs_tracker_query_storage_list_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, true); +} + +/* +boolean fastdfs_tracker_delete_storage(string group_name, string storage_ip) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_tracker_delete_storage) +{ + php_fdfs_tracker_delete_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context); +} + +/* +array fastdfs_storage_upload_by_filename(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_by_filename) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_FILE, false); +} + +/* +string fastdfs_storage_upload_by_filename1(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_by_filename1) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_FILE, true); +} + +/* +array fastdfs_storage_upload_by_filebuff(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_by_filebuff) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string fastdfs_storage_upload_by_filebuff1(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_by_filebuff1) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array fastdfs_storage_upload_by_callback(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_by_callback) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string fastdfs_storage_upload_by_callback1(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_by_callback1) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +boolean fastdfs_storage_append_by_filename(string local_filename, + string group_name, appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_append_by_filename) +{ + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_FILE, false); +} + +/* +boolean fastdfs_storage_append_by_filename1(string local_filename, + string appender_file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_append_by_filename1) +{ + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_FILE, true); +} + +/* +boolean fastdfs_storage_append_by_filebuff(string file_buff, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_append_by_filebuff) +{ + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_BUFF, false); +} + +/* +boolean fastdfs_storage_append_by_filebuff1(string file_buff, + string appender_file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_append_by_filebuff1) +{ + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_BUFF, true); +} + +/* +boolean fastdfs_storage_append_by_callback(array callback_array, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_append_by_callback) +{ + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +boolean fastdfs_storage_append_by_callback1(array callback_array, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_append_by_callback1) +{ + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +boolean fastdfs_storage_modify_by_filename(string local_filename, + long file_offset, string group_name, appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_modify_by_filename) +{ + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_FILE, false); +} + +/* +boolean fastdfs_storage_modify_by_filename1(string local_filename, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_modify_by_filename1) +{ + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_FILE, true); +} + +/* +boolean fastdfs_storage_modify_by_filebuff(string file_buff, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_modify_by_filebuff) +{ + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_BUFF, false); +} + +/* +boolean fastdfs_storage_modify_by_filebuff1(string file_buff, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_modify_by_filebuff1) +{ + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_BUFF, true); +} + +/* +boolean fastdfs_storage_modify_by_callback(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_modify_by_callback) +{ + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +boolean fastdfs_storage_modify_by_callback1(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_modify_by_callback1) +{ + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +array fastdfs_storage_upload_appender_by_filename(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filename) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_FILE, false); +} + +/* +string fastdfs_storage_upload_appender_by_filename1(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filename1) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_FILE, true); +} + +/* +array fastdfs_storage_upload_appender_by_filebuff(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filebuff) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string fastdfs_storage_upload_appender_by_filebuff1(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filebuff1) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array fastdfs_storage_upload_appender_by_callback(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_callback) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string fastdfs_storage_upload_appender_by_callback1(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_callback1) +{ + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +string/array fastdfs_storage_upload_slave_by_filename(string local_filename, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +return string/array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filename) +{ + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + FDFS_UPLOAD_BY_FILE, false); +} + +/* +string fastdfs_storage_upload_slave_by_filename1(string local_filename, + string master_file_id, string prefix_name [, string file_ext_name, + string meta_list, array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filename1) +{ + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + FDFS_UPLOAD_BY_FILE, true); +} + +/* +array fastdfs_storage_upload_slave_by_filebuff(string file_buff, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filebuff) +{ + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string fastdfs_storage_upload_slave_by_filebuff1(string file_buff, + string master_file_id, string prefix_name [, string file_ext_name, + string meta_list, array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filebuff1) +{ + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array fastdfs_storage_upload_slave_by_callback(array callback_array, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_callback) +{ + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string fastdfs_storage_upload_slave_by_callback1(array callback_array, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, array meta_list, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_callback1) +{ + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, \ + FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +boolean fastdfs_storage_delete_file(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_delete_file) +{ + php_fdfs_storage_delete_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, false); +} + +/* +boolean fastdfs_storage_delete_file1(string file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_delete_file1) +{ + php_fdfs_storage_delete_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, true); +} + +/* +boolean fastdfs_storage_truncate_file(string group_name, + string appender_filename [, long truncated_file_size = 0, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_truncate_file) +{ + php_fdfs_storage_truncate_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, false); +} + +/* +boolean fastdfs_storage_truncate_file1(string appender_file_id + [, long truncated_file_size = 0, array tracker_server, + array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_truncate_file1) +{ + php_fdfs_storage_truncate_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &php_context, true); +} + +/* +string fastdfs_storage_download_file_to_buff(string group_name, + string remote_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return file content for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_download_file_to_buff) +{ + php_fdfs_storage_download_file_to_buff_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +string fastdfs_storage_download_file_to_buff1(string file_id + [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return file content for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_download_file_to_buff1) +{ + php_fdfs_storage_download_file_to_buff_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +boolean fastdfs_storage_download_file_to_callback(string group_name, + string remote_filename, array download_callback [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_download_file_to_callback) +{ + php_fdfs_storage_download_file_to_callback_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +boolean fastdfs_storage_download_file_to_callback1(string file_id, + array download_callback [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_download_file_to_callback1) +{ + php_fdfs_storage_download_file_to_callback_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +boolean fastdfs_storage_download_file_to_file(string group_name, + string remote_filename, string local_filename [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_download_file_to_file) +{ + php_fdfs_storage_download_file_to_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +boolean fastdfs_storage_download_file_to_file1(string file_id, + string local_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_download_file_to_file1) +{ + php_fdfs_storage_download_file_to_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +boolean fastdfs_storage_set_metadata(string group_name, string remote_filename, + array meta_list [, string op_type, array tracker_server, + array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_set_metadata) +{ + php_fdfs_storage_set_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +boolean fastdfs_storage_set_metadata1(string file_id, array meta_list + [, string op_type, array tracker_server, array storage_server]) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_set_metadata1) +{ + php_fdfs_storage_set_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +array fastdfs_storage_get_metadata(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_get_metadata) +{ + php_fdfs_storage_get_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +array fastdfs_storage_get_metadata1(string file_id + [, array tracker_server, array storage_server]) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_storage_get_metadata1) +{ + php_fdfs_storage_get_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +boolean fastdfs_storage_file_exist(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +return true for exist, false for not exist +*/ +ZEND_FUNCTION(fastdfs_storage_file_exist) +{ + php_fdfs_storage_file_exist_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +boolean fastdfs_storage_file_exist1(string file_id + [, array tracker_server, array storage_server]) +return true for exist, false for not exist +*/ +ZEND_FUNCTION(fastdfs_storage_file_exist1) +{ + php_fdfs_storage_file_exist_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +string fastdfs_http_gen_token(string file_id, int timestamp) +return token string for success, false for error +*/ +ZEND_FUNCTION(fastdfs_http_gen_token) +{ + php_fdfs_http_gen_token_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +/* +array fastdfs_get_file_info(string group_name, string remote_filename) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_get_file_info) +{ + php_fdfs_get_file_info_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false); +} + +/* +array fastdfs_get_file_info1(string file_id) +return array for success, false for error +*/ +ZEND_FUNCTION(fastdfs_get_file_info1) +{ + php_fdfs_get_file_info_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true); +} + +/* +bool fastdfs_send_data(int sock, string buff) +return true for success, false for error +*/ +ZEND_FUNCTION(fastdfs_send_data) +{ + php_fdfs_send_data_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +/* +string fastdfs_gen_slave_filename(string master_filename, string prefix_name + [, string file_ext_name]) +return slave filename string for success, false for error +*/ +ZEND_FUNCTION(fastdfs_gen_slave_filename) +{ + php_fdfs_gen_slave_filename_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context); +} + +static void php_fdfs_close(php_fdfs_t *i_obj TSRMLS_DC) +{ + if (i_obj->context.pTrackerGroup == NULL) + { + return; + } + + if (i_obj->context.pTrackerGroup != i_obj->pConfigInfo->pTrackerGroup) + { + tracker_close_all_connections_ex(i_obj->context.pTrackerGroup); + } +} + +/* constructor/destructor */ +static void php_fdfs_destroy(php_fdfs_t *i_obj TSRMLS_DC) +{ + php_fdfs_close(i_obj TSRMLS_CC); + if (i_obj->context.pTrackerGroup != NULL && i_obj->context.pTrackerGroup != \ + i_obj->pConfigInfo->pTrackerGroup) + { + fdfs_client_destroy_ex(i_obj->context.pTrackerGroup); + efree(i_obj->context.pTrackerGroup); + i_obj->context.pTrackerGroup = NULL; + } + + efree(i_obj); +} + +ZEND_RSRC_DTOR_FUNC(php_fdfs_dtor) +{ + if (rsrc->ptr != NULL) + { + php_fdfs_t *i_obj = (php_fdfs_t *)rsrc->ptr; + php_fdfs_destroy(i_obj TSRMLS_CC); + rsrc->ptr = NULL; + } +} + +/* FastDFS::__construct([int config_index = 0, bool bMultiThread = false]) + Creates a FastDFS object */ +static PHP_METHOD(FastDFS, __construct) +{ + long config_index; + bool bMultiThread; + zval *object = getThis(); + php_fdfs_t *i_obj; + + config_index = 0; + bMultiThread = false; + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|lb", \ + &config_index, &bMultiThread) == FAILURE) + { + logError("file: "__FILE__", line: %d, " \ + "zend_parse_parameters fail!", __LINE__); + ZVAL_NULL(object); + return; + } + + if (config_index < 0 || config_index >= config_count) + { + logError("file: "__FILE__", line: %d, " \ + "invalid config_index: %ld < 0 || >= %d", \ + __LINE__, config_index, config_count); + ZVAL_NULL(object); + return; + } + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + i_obj->pConfigInfo = config_list + config_index; + i_obj->context.err_no = 0; + if (bMultiThread) + { + i_obj->context.pTrackerGroup = (TrackerServerGroup *)emalloc( \ + sizeof(TrackerServerGroup)); + if (i_obj->context.pTrackerGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail!", __LINE__, \ + (int)sizeof(TrackerServerGroup)); + ZVAL_NULL(object); + return; + } + + if (fdfs_copy_tracker_group(i_obj->context.pTrackerGroup, \ + i_obj->pConfigInfo->pTrackerGroup) != 0) + { + ZVAL_NULL(object); + return; + } + } + else + { + i_obj->context.pTrackerGroup = i_obj->pConfigInfo->pTrackerGroup; + } +} + +/* +array FastDFS::tracker_get_connection() +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_get_connection) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_get_connection_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +boolean FastDFS::tracker_make_all_connections() +return true for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_make_all_connections) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_make_all_connections_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context)); +} + +/* +boolean FastDFS::tracker_close_all_connections() +return true for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_close_all_connections) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_close_all_connections_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context)); +} + +/* +array FastDFS::connect_server(string ip_addr, int port) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, connect_server) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_connect_server_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +boolean FastDFS::disconnect_server(array serverInfo) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, disconnect_server) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_disconnect_server_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +boolean FastDFS::active_test(array serverInfo) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, active_test) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fastdfs_active_test_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +array FastDFS::tracker_list_groups([string group_name, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_list_groups) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_list_groups_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +array FastDFS::tracker_query_storage_store([string group_name, + array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_store) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_query_storage_store_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +array FastDFS::tracker_query_storage_store_list([string group_name, + array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_store_list) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_query_storage_store_list_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context)); +} + +/* +array FastDFS::tracker_query_storage_update(string group_name, + string remote_filename [, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_update) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE, false); +} + +/* +array FastDFS::tracker_query_storage_fetch(string group_name, + string remote_filename [, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_fetch) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE, false); +} + +/* +array FastDFS::tracker_query_storage_list(string group_name, + string remote_filename [, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_list) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_query_storage_list_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +boolean FastDFS::tracker_delete_storage(string group_name, string storage_ip) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_delete_storage) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_delete_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context)); +} + +/* +array FastDFS::tracker_query_storage_update1(string file_id + [, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_update1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE, true); +} + +/* +array FastDFS::tracker_query_storage_fetch1(string file_id + [, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_fetch1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_do_query_storage_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE, true); +} + +/* +array FastDFS::tracker_query_storage_list1(string file_id + [, array tracker_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, tracker_query_storage_list1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_tracker_query_storage_list_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +array FastDFS::storage_upload_by_filename(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_by_filename) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_FILE, false); +} + +/* +string FastDFS::storage_upload_by_filename1(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_by_filename1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_FILE, true); +} + +/* +array FastDFS::storage_upload_by_filebuff(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_by_filebuff) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string FastDFS::storage_upload_by_filebuff1(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_by_filebuff1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array FastDFS::storage_upload_by_callback(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_by_callback) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string FastDFS::storage_upload_by_callback1(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_by_callback1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +boolean FastDFS::storage_append_by_filename(string local_filename, + string group_name, appender_filename + [, array tracker_server, array storage_server]) +return string/array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_append_by_filename) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_FILE, false); +} + +/* +string FastDFS::storage_upload_by_filename1(string local_filename, + string appender_file_id + [, array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_append_by_filename1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_FILE, true); +} + +/* +array FastDFS::storage_append_by_filebuff(string file_buff, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_append_by_filebuff) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string FastDFS::storage_append_by_filebuff1(string file_buff, + string appender_file_id + [, array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_append_by_filebuff1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array FastDFS::storage_append_by_callback(array callback_array, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_append_by_callback) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string FastDFS::storage_append_by_callback1(array callback_array, + string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_append_by_callback1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_append_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_CALLBACK, true); +} + + +/* +boolean FastDFS::storage_modify_by_filename(string local_filename, + long file_offset, string group_name, appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_modify_by_filename) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_FILE, false); +} + +/* +boolean FastDFS::storage_modify_by_filename1(string local_filename, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_modify_by_filename1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_FILE, true); +} + +/* +boolean FastDFS::storage_modify_by_filebuff(string file_buff, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_modify_by_filebuff) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_BUFF, false); +} + +/* +boolean FastDFS::storage_modify_by_filebuff1(string file_buff, + long file_offset, string appender_file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_modify_by_filebuff1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_BUFF, true); +} + +/* +boolean FastDFS::storage_modify_by_callback(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_modify_by_callback) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +boolean FastDFS::storage_modify_by_callback1(array callback_array, + long file_offset, string group_name, string appender_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_modify_by_callback1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_modify_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +array FastDFS::storage_upload_appender_by_filename(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_appender_by_filename) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_FILE, false); +} + +/* +string FastDFS::storage_upload_appender_by_filename1(string local_filename, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_appender_by_filename1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_FILE, true); +} + +/* +array FastDFS::storage_upload_appender_by_filebuff(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_appender_by_filebuff) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string FastDFS::storage_upload_appender_by_filebuff1(string file_buff, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_appender_by_filebuff1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array FastDFS::storage_upload_appender_by_callback(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_appender_by_callback) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string FastDFS::storage_upload_appender_by_callback1(array callback_array, + [string file_ext_name, string meta_list, string group_name, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_appender_by_callback1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, \ + FDFS_UPLOAD_BY_CALLBACK, true); +} + + +/* +array FastDFS::storage_upload_slave_by_filename(string local_filename, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, string meta_list, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_slave_by_filename) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + FDFS_UPLOAD_BY_FILE, false); +} + +/* +string FastDFS::storage_upload_slave_by_filename1(string local_filename, + string master_file_id, string prefix_name + [, string file_ext_name, string meta_list, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_slave_by_filename1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + FDFS_UPLOAD_BY_FILE, true); +} + +/* +array FastDFS::storage_upload_slave_by_filebuff(string file_buff, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, string meta_list, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_slave_by_filebuff) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + FDFS_UPLOAD_BY_BUFF, false); +} + +/* +string FastDFS::storage_upload_slave_by_filebuff1(string file_buff, + string master_file_id, string prefix_name [, string file_ext_name, + string meta_list, array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_slave_by_filebuff1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + FDFS_UPLOAD_BY_BUFF, true); +} + +/* +array FastDFS::storage_upload_slave_by_callback(array callback_array, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, string meta_list, + array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_slave_by_callback) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + FDFS_UPLOAD_BY_CALLBACK, false); +} + +/* +string FastDFS::storage_upload_slave_by_callback1(array callback_array, + string group_name, string master_filename, string prefix_name + [, string file_ext_name, string meta_list, + array tracker_server, array storage_server]) +return file_id for success, false for error +*/ +PHP_METHOD(FastDFS, storage_upload_slave_by_callback1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_upload_slave_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), \ + FDFS_UPLOAD_BY_CALLBACK, true); +} + +/* +boolean FastDFS::storage_delete_file(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_delete_file) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_delete_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), false); +} + +/* +boolean FastDFS::storage_delete_file1(string file_id + [, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_delete_file1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_delete_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), true); +} + +/* +boolean FastDFS::storage_truncate_file(string group_name, + string remote_filename [, long truncated_file_size = 0, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_truncate_file) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_truncate_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), false); +} + +/* +boolean FastDFS::storage_truncate_file1(string file_id + [, long truncated_file_size = 0, array tracker_server, + array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_truncate_file1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_truncate_file_impl(INTERNAL_FUNCTION_PARAM_PASSTHRU, \ + &(i_obj->context), true); +} + +/* +string FastDFS::storage_download_file_to_buff(string group_name, + string remote_filename [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return file content for success, false for error +*/ +PHP_METHOD(FastDFS, storage_download_file_to_buff) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_download_file_to_buff_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +string FastDFS::storage_download_file_to_buff1(string file_id + [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return file content for success, false for error +*/ +PHP_METHOD(FastDFS, storage_download_file_to_buff1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_download_file_to_buff_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +boolean FastDFS::storage_download_file_to_callback(string group_name, + string remote_filename, array download_callback [, long file_offset, + long download_bytes, array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_download_file_to_callback) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_download_file_to_callback_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +boolean FastDFS::storage_download_file_to_callback1(string file_id, + array download_callback [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_download_file_to_callback1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_download_file_to_callback_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +boolean FastDFS::storage_download_file_to_file(string group_name, + string remote_filename, string local_filename + [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_download_file_to_file) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_download_file_to_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +boolean FastDFS::storage_download_file_to_file1(string file_id, + string local_filename, [, long file_offset, long download_bytes, + array tracker_server, array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_download_file_to_file1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_download_file_to_file_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +boolean FastDFS::storage_set_metadata(string group_name, string remote_filename, + array meta_list [, string op_type, array tracker_server, + array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_set_metadata) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_set_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +boolean FastDFS::storage_set_metadata1(string file_id, + array meta_list [, string op_type, array tracker_server, + array storage_server]) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, storage_set_metadata1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_set_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +array FastDFS::storage_get_metadata(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_get_metadata) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_get_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +array FastDFS::storage_get_metadata1(string file_id + [, array tracker_server, array storage_server]) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, storage_get_metadata1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_get_metadata_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +boolean FastDFS::storage_file_exist(string group_name, string remote_filename + [, array tracker_server, array storage_server]) +return true for exist, false for not exist +*/ +PHP_METHOD(FastDFS, storage_file_exist) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_file_exist_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +boolean FastDFS::storage_file_exist1(string file_id + [, array tracker_server, array storage_server]) +return true for exist, false for not exist +*/ +PHP_METHOD(FastDFS, storage_file_exist1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_storage_file_exist_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +long FastDFS::get_last_error_no() +return last error no +*/ +PHP_METHOD(FastDFS, get_last_error_no) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + RETURN_LONG(i_obj->context.err_no); +} + +/* +string FastDFS::get_last_error_info() +return last error info +*/ +PHP_METHOD(FastDFS, get_last_error_info) +{ + char *error_info; + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + error_info = STRERROR(i_obj->context.err_no); + RETURN_STRINGL(error_info, strlen(error_info), 1); +} + +/* +string FastDFS::http_gen_token(string file_id, int timestamp) +return token string for success, false for error +*/ +PHP_METHOD(FastDFS, http_gen_token) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_http_gen_token_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context)); +} + +/* +array FastDFS::get_file_info(string group_name, string remote_filename) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, get_file_info) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_get_file_info_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false); +} + +/* +array FastDFS::get_file_info1(string file_id) +return array for success, false for error +*/ +PHP_METHOD(FastDFS, get_file_info1) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_get_file_info_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true); +} + +/* +bool FastDFS::send_data(int sock, string buff) +return true for success, false for error +*/ +PHP_METHOD(FastDFS, send_data) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_send_data_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context)); +} + +/* +string FastDFS::gen_slave_filename(string master_filename, string prefix_name + [, string file_ext_name]) +return slave filename string for success, false for error +*/ +PHP_METHOD(FastDFS, gen_slave_filename) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_gen_slave_filename_impl( \ + INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context)); +} + +/* +void FastDFS::close() +*/ +PHP_METHOD(FastDFS, close) +{ + zval *object = getThis(); + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *) zend_object_store_get_object(object TSRMLS_CC); + php_fdfs_close(i_obj TSRMLS_CC); +} + +ZEND_BEGIN_ARG_INFO_EX(arginfo___construct, 0, 0, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_get_connection, 0, 0, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_make_all_connections, 0, 0, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_close_all_connections, 0, 0, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_connect_server, 0, 0, 2) +ZEND_ARG_INFO(0, ip_addr) +ZEND_ARG_INFO(0, port) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_disconnect_server, 0, 0, 1) +ZEND_ARG_INFO(0, server_info) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_active_test, 0, 0, 1) +ZEND_ARG_INFO(0, server_info) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_list_groups, 0, 0, 0) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_store, 0, 0, 0) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_store_list, 0, 0, 0) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_update, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_fetch, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_list, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_update1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_fetch1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_query_storage_list1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_tracker_delete_storage, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, storage_ip) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_by_filename, 0, 0, 1) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_by_filename1, 0, 0, 1) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_by_filebuff, 0, 0, 1) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_by_filebuff1, 0, 0, 1) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_by_callback, 0, 0, 1) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_by_callback1, 0, 0, 1) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_append_by_filename, 0, 0, 3) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, appender_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_append_by_filename1, 0, 0, 2) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, appender_file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_append_by_filebuff, 0, 0, 3) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, appender_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_append_by_filebuff1, 0, 0, 2) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, appender_file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_append_by_callback, 0, 0, 3) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, appender_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_append_by_callback1, 0, 0, 2) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, appender_file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_modify_by_filename, 0, 0, 3) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, appender_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_modify_by_filename1, 0, 0, 2) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, appender_file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_modify_by_filebuff, 0, 0, 3) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, appender_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_modify_by_filebuff1, 0, 0, 2) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, appender_file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_modify_by_callback, 0, 0, 3) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, appender_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_modify_by_callback1, 0, 0, 2) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, appender_file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_filename, 0, 0, 1) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_filename1, 0, 0, 1) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_filebuff, 0, 0, 1) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_filebuff1, 0, 0, 1) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_callback, 0, 0, 1) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_callback1, 0, 0, 1) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_slave_by_filename, 0, 0, 4) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, master_filename) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_slave_by_filename1, 0, 0, 3) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, master_file_id) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_slave_by_filebuff, 0, 0, 4) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, master_filename) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_slave_by_filebuff1, 0, 0, 3) +ZEND_ARG_INFO(0, file_buff) +ZEND_ARG_INFO(0, master_file_id) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_slave_by_callback, 0, 0, 4) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, master_filename) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_slave_by_callback1, 0, 0, 3) +ZEND_ARG_INFO(0, callback_array) +ZEND_ARG_INFO(0, master_file_id) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_delete_file, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_delete_file1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_truncate_file, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, truncated_file_size) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_truncate_file1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, truncated_file_size) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_download_file_to_buff, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, file_offset) +ZEND_ARG_INFO(0, download_bytes) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_download_file_to_buff1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, file_offset) +ZEND_ARG_INFO(0, download_bytes) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_download_file_to_callback, 0, 0, 3) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, download_callback) +ZEND_ARG_INFO(0, file_offset) +ZEND_ARG_INFO(0, download_bytes) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_download_file_to_callback1, 0, 0, 2) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, download_callback) +ZEND_ARG_INFO(0, file_offset) +ZEND_ARG_INFO(0, download_bytes) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_download_file_to_file, 0, 0, 3) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, file_offset) +ZEND_ARG_INFO(0, download_bytes) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_download_file_to_file1, 0, 0, 2) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, local_filename) +ZEND_ARG_INFO(0, file_offset) +ZEND_ARG_INFO(0, download_bytes) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_set_metadata, 0, 0, 3) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, op_type) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_set_metadata1, 0, 0, 2) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, meta_list) +ZEND_ARG_INFO(0, op_type) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_get_metadata, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_get_metadata1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_file_exist, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_file_exist1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, tracker_server) +ZEND_ARG_INFO(0, storage_server) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_get_last_error_no, 0, 0, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_get_last_error_info, 0, 0, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_http_gen_token, 0, 0, 2) +ZEND_ARG_INFO(0, file_id) +ZEND_ARG_INFO(0, timestamp) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_get_file_info, 0, 0, 2) +ZEND_ARG_INFO(0, group_name) +ZEND_ARG_INFO(0, remote_filename) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_get_file_info1, 0, 0, 1) +ZEND_ARG_INFO(0, file_id) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_send_data, 0, 0, 2) +ZEND_ARG_INFO(0, sock) +ZEND_ARG_INFO(0, buff) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_gen_slave_filename, 0, 0, 2) +ZEND_ARG_INFO(0, master_filename) +ZEND_ARG_INFO(0, prefix_name) +ZEND_ARG_INFO(0, file_ext_name) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_close, 0, 0, 0) +ZEND_END_ARG_INFO() + +/* {{{ fdfs_class_methods */ +#define FDFS_ME(name, args) PHP_ME(FastDFS, name, args, ZEND_ACC_PUBLIC) +static zend_function_entry fdfs_class_methods[] = { + FDFS_ME(__construct, arginfo___construct) + FDFS_ME(tracker_get_connection, arginfo_tracker_get_connection) + FDFS_ME(tracker_make_all_connections, arginfo_tracker_make_all_connections) + FDFS_ME(tracker_close_all_connections,arginfo_tracker_close_all_connections) + FDFS_ME(active_test, arginfo_active_test) + FDFS_ME(connect_server, arginfo_connect_server) + FDFS_ME(disconnect_server, arginfo_disconnect_server) + FDFS_ME(tracker_list_groups, arginfo_tracker_list_groups) + FDFS_ME(tracker_query_storage_store, arginfo_tracker_query_storage_store) + FDFS_ME(tracker_query_storage_store_list, arginfo_tracker_query_storage_store_list) + FDFS_ME(tracker_query_storage_update, arginfo_tracker_query_storage_update) + FDFS_ME(tracker_query_storage_fetch, arginfo_tracker_query_storage_fetch) + FDFS_ME(tracker_query_storage_list, arginfo_tracker_query_storage_list) + FDFS_ME(tracker_query_storage_update1,arginfo_tracker_query_storage_update1) + FDFS_ME(tracker_query_storage_fetch1, arginfo_tracker_query_storage_fetch1) + FDFS_ME(tracker_query_storage_list1, arginfo_tracker_query_storage_list1) + FDFS_ME(tracker_delete_storage, arginfo_tracker_delete_storage) + FDFS_ME(storage_upload_by_filename, arginfo_storage_upload_by_filename) + FDFS_ME(storage_upload_by_filename1, arginfo_storage_upload_by_filename1) + FDFS_ME(storage_upload_by_filebuff, arginfo_storage_upload_by_filebuff) + FDFS_ME(storage_upload_by_filebuff1, arginfo_storage_upload_by_filebuff1) + FDFS_ME(storage_upload_by_callback, arginfo_storage_upload_by_callback) + FDFS_ME(storage_upload_by_callback1, arginfo_storage_upload_by_callback1) + FDFS_ME(storage_append_by_filename, arginfo_storage_append_by_filename) + FDFS_ME(storage_append_by_filename1, arginfo_storage_append_by_filename1) + FDFS_ME(storage_append_by_filebuff, arginfo_storage_append_by_filebuff) + FDFS_ME(storage_append_by_filebuff1, arginfo_storage_append_by_filebuff1) + FDFS_ME(storage_append_by_callback, arginfo_storage_append_by_callback) + FDFS_ME(storage_append_by_callback1, arginfo_storage_append_by_callback1) + FDFS_ME(storage_modify_by_filename, arginfo_storage_modify_by_filename) + FDFS_ME(storage_modify_by_filename1, arginfo_storage_modify_by_filename1) + FDFS_ME(storage_modify_by_filebuff, arginfo_storage_modify_by_filebuff) + FDFS_ME(storage_modify_by_filebuff1, arginfo_storage_modify_by_filebuff1) + FDFS_ME(storage_modify_by_callback, arginfo_storage_modify_by_callback) + FDFS_ME(storage_modify_by_callback1, arginfo_storage_modify_by_callback1) + FDFS_ME(storage_upload_appender_by_filename, arginfo_storage_upload_appender_by_filename) + FDFS_ME(storage_upload_appender_by_filename1, arginfo_storage_upload_appender_by_filename1) + FDFS_ME(storage_upload_appender_by_filebuff, arginfo_storage_upload_appender_by_filebuff) + FDFS_ME(storage_upload_appender_by_filebuff1, arginfo_storage_upload_appender_by_filebuff1) + FDFS_ME(storage_upload_appender_by_callback, arginfo_storage_upload_appender_by_callback) + FDFS_ME(storage_upload_appender_by_callback1, arginfo_storage_upload_appender_by_callback1) + FDFS_ME(storage_upload_slave_by_filename, arginfo_storage_upload_slave_by_filename) + FDFS_ME(storage_upload_slave_by_filename1, arginfo_storage_upload_slave_by_filename1) + FDFS_ME(storage_upload_slave_by_filebuff, arginfo_storage_upload_slave_by_filebuff) + FDFS_ME(storage_upload_slave_by_filebuff1, arginfo_storage_upload_slave_by_filebuff1) + FDFS_ME(storage_upload_slave_by_callback, arginfo_storage_upload_slave_by_callback) + FDFS_ME(storage_upload_slave_by_callback1, arginfo_storage_upload_slave_by_callback1) + FDFS_ME(storage_delete_file, arginfo_storage_delete_file) + FDFS_ME(storage_delete_file1, arginfo_storage_delete_file1) + FDFS_ME(storage_truncate_file, arginfo_storage_truncate_file) + FDFS_ME(storage_truncate_file1, arginfo_storage_truncate_file1) + FDFS_ME(storage_download_file_to_buff, arginfo_storage_download_file_to_buff) + FDFS_ME(storage_download_file_to_buff1,arginfo_storage_download_file_to_buff1) + FDFS_ME(storage_download_file_to_file, arginfo_storage_download_file_to_file) + FDFS_ME(storage_download_file_to_file1,arginfo_storage_download_file_to_file1) + FDFS_ME(storage_download_file_to_callback, arginfo_storage_download_file_to_callback) + FDFS_ME(storage_download_file_to_callback1, arginfo_storage_download_file_to_callback1) + FDFS_ME(storage_set_metadata, arginfo_storage_set_metadata) + FDFS_ME(storage_set_metadata1, arginfo_storage_set_metadata1) + FDFS_ME(storage_get_metadata, arginfo_storage_get_metadata) + FDFS_ME(storage_get_metadata1, arginfo_storage_get_metadata1) + FDFS_ME(storage_file_exist, arginfo_storage_file_exist) + FDFS_ME(storage_file_exist1, arginfo_storage_file_exist1) + FDFS_ME(get_last_error_no, arginfo_get_last_error_no) + FDFS_ME(get_last_error_info, arginfo_get_last_error_info) + FDFS_ME(http_gen_token, arginfo_http_gen_token) + FDFS_ME(get_file_info, arginfo_get_file_info) + FDFS_ME(get_file_info1, arginfo_get_file_info1) + FDFS_ME(send_data, arginfo_send_data) + FDFS_ME(gen_slave_filename, arginfo_gen_slave_filename) + FDFS_ME(close, arginfo_close) + { NULL, NULL, NULL } +}; +#undef FDFS_ME +/* }}} */ + +static void php_fdfs_free_storage(php_fdfs_t *i_obj TSRMLS_DC) +{ + zend_object_std_dtor(&i_obj->zo TSRMLS_CC); + php_fdfs_destroy(i_obj TSRMLS_CC); +} + +zend_object_value php_fdfs_new(zend_class_entry *ce TSRMLS_DC) +{ + zend_object_value retval; + php_fdfs_t *i_obj; + + i_obj = (php_fdfs_t *)ecalloc(1, sizeof(php_fdfs_t)); + + zend_object_std_init(&i_obj->zo, ce TSRMLS_CC); + retval.handle = zend_objects_store_put(i_obj, \ + (zend_objects_store_dtor_t)zend_objects_destroy_object, \ + (zend_objects_free_object_storage_t)php_fdfs_free_storage, \ + NULL TSRMLS_CC); + retval.handlers = zend_get_std_object_handlers(); + + return retval; +} + +PHP_FASTDFS_API zend_class_entry *php_fdfs_get_ce(void) +{ + return fdfs_ce; +} + +PHP_FASTDFS_API zend_class_entry *php_fdfs_get_exception(void) +{ + return fdfs_exception_ce; +} + +PHP_FASTDFS_API zend_class_entry *php_fdfs_get_exception_base(int root TSRMLS_DC) +{ +#if HAVE_SPL + if (!root) + { + if (!spl_ce_RuntimeException) + { + zend_class_entry **pce; + zend_class_entry ***ppce; + + ppce = &pce; + if (zend_hash_find(CG(class_table), "runtimeexception", + sizeof("RuntimeException"), (void **) ppce) == SUCCESS) + { + spl_ce_RuntimeException = *pce; + return *pce; + } + } + else + { + return spl_ce_RuntimeException; + } + } +#endif +#if (PHP_MAJOR_VERSION == 5) && (PHP_MINOR_VERSION < 2) + return zend_exception_get_default(); +#else + return zend_exception_get_default(TSRMLS_C); +#endif +} + + +static int load_config_files() +{ + #define ITEM_NAME_CONF_COUNT "fastdfs_client.tracker_group_count" + #define ITEM_NAME_CONF_FILE "fastdfs_client.tracker_group" + #define ITEM_NAME_BASE_PATH "fastdfs_client.base_path" + #define ITEM_NAME_CONNECT_TIMEOUT "fastdfs_client.connect_timeout" + #define ITEM_NAME_NETWORK_TIMEOUT "fastdfs_client.network_timeout" + #define ITEM_NAME_LOG_LEVEL "fastdfs_client.log_level" + #define ITEM_NAME_LOG_FILENAME "fastdfs_client.log_filename" + #define ITEM_NAME_ANTI_STEAL_SECRET_KEY "fastdfs_client.http.anti_steal_secret_key" + #define ITEM_NAME_USE_CONN_POOL "fastdfs_client.use_connection_pool" + #define ITEM_NAME_CONN_POOL_MAX_IDLE_TIME "fastdfs_client.connection_pool_max_idle_time" + + zval conf_c; + zval base_path; + zval connect_timeout; + zval network_timeout; + zval log_level; + zval anti_steal_secret_key; + zval log_filename; + zval conf_filename; + zval use_conn_pool; + zval conn_pool_max_idle_time; + char *pAntiStealSecretKey; + char szItemName[sizeof(ITEM_NAME_CONF_FILE) + 10]; + int nItemLen; + FDFSConfigInfo *pConfigInfo; + FDFSConfigInfo *pConfigEnd; + int result; + + if (zend_get_configuration_directive(ITEM_NAME_CONF_COUNT, + sizeof(ITEM_NAME_CONF_COUNT), &conf_c) == SUCCESS) + { + config_count = atoi(conf_c.value.str.val); + if (config_count <= 0) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "fastdfs_client.ini, config_count: %d <= 0!\n",\ + __LINE__, config_count); + return EINVAL; + } + } + else + { + config_count = 1; + } + + if (zend_get_configuration_directive(ITEM_NAME_BASE_PATH, \ + sizeof(ITEM_NAME_BASE_PATH), &base_path) != SUCCESS) + { + strcpy(g_fdfs_base_path, "/tmp"); + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "fastdht_client.ini does not have item " \ + "\"%s\", set to %s!", __LINE__, + ITEM_NAME_BASE_PATH, g_fdfs_base_path); + } + else + { + snprintf(g_fdfs_base_path, sizeof(g_fdfs_base_path), "%s", \ + base_path.value.str.val); + chopPath(g_fdfs_base_path); + } + + if (!fileExists(g_fdfs_base_path)) + { + logError("\"%s\" can't be accessed, error info: %s", \ + g_fdfs_base_path, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + if (!isDir(g_fdfs_base_path)) + { + logError("\"%s\" is not a directory!", g_fdfs_base_path); + return ENOTDIR; + } + + if (zend_get_configuration_directive(ITEM_NAME_CONNECT_TIMEOUT, \ + sizeof(ITEM_NAME_CONNECT_TIMEOUT), \ + &connect_timeout) == SUCCESS) + { + g_fdfs_connect_timeout = atoi(connect_timeout.value.str.val); + if (g_fdfs_connect_timeout <= 0) + { + g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT; + } + } + else + { + g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT; + } + + if (zend_get_configuration_directive(ITEM_NAME_NETWORK_TIMEOUT, \ + sizeof(ITEM_NAME_NETWORK_TIMEOUT), \ + &network_timeout) == SUCCESS) + { + g_fdfs_network_timeout = atoi(network_timeout.value.str.val); + if (g_fdfs_network_timeout <= 0) + { + g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT; + } + } + else + { + g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT; + } + + if (zend_get_configuration_directive(ITEM_NAME_LOG_LEVEL, \ + sizeof(ITEM_NAME_LOG_LEVEL), \ + &log_level) == SUCCESS) + { + set_log_level(log_level.value.str.val); + } + + + if (zend_get_configuration_directive(ITEM_NAME_LOG_FILENAME, \ + sizeof(ITEM_NAME_LOG_FILENAME), \ + &log_filename) == SUCCESS) + { + if (log_filename.value.str.len > 0) + { + log_set_filename(log_filename.value.str.val); + } + } + + if (zend_get_configuration_directive(ITEM_NAME_ANTI_STEAL_SECRET_KEY, \ + sizeof(ITEM_NAME_ANTI_STEAL_SECRET_KEY), \ + &anti_steal_secret_key) == SUCCESS) + { + pAntiStealSecretKey = anti_steal_secret_key.value.str.val; + } + else + { + pAntiStealSecretKey = ""; + } + buffer_strcpy(&g_anti_steal_secret_key, pAntiStealSecretKey); + + config_list = (FDFSConfigInfo *)malloc(sizeof(FDFSConfigInfo) * \ + config_count); + if (config_list == NULL) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "malloc %d bytes fail!\n",\ + __LINE__, (int)sizeof(FDFSConfigInfo) * config_count); + return errno != 0 ? errno : ENOMEM; + } + + pConfigEnd = config_list + config_count; + for (pConfigInfo=config_list; pConfigInfopTrackerGroup = &g_tracker_group; + } + else + { + pConfigInfo->pTrackerGroup = (TrackerServerGroup *)malloc( \ + sizeof(TrackerServerGroup)); + if (pConfigInfo->pTrackerGroup == NULL) + { + fprintf(stderr, "file: "__FILE__", line: %d, " \ + "malloc %d bytes fail!\n", \ + __LINE__, (int)sizeof(TrackerServerGroup)); + return errno != 0 ? errno : ENOMEM; + } + } + + if ((result=fdfs_load_tracker_group(pConfigInfo->pTrackerGroup, + conf_filename.value.str.val)) != 0) + { + return result; + } + } + + + if (zend_get_configuration_directive(ITEM_NAME_USE_CONN_POOL, + sizeof(ITEM_NAME_USE_CONN_POOL), &use_conn_pool) == SUCCESS) + { + char *use_conn_pool_str; + + use_conn_pool_str = use_conn_pool.value.str.val; + if (strcasecmp(use_conn_pool_str, "yes") == 0 || + strcasecmp(use_conn_pool_str, "on") == 0 || + strcasecmp(use_conn_pool_str, "true") == 0 || + strcmp(use_conn_pool_str, "1") == 0) + { + if (zend_get_configuration_directive( \ + ITEM_NAME_CONN_POOL_MAX_IDLE_TIME, \ + sizeof(ITEM_NAME_CONN_POOL_MAX_IDLE_TIME), \ + &conn_pool_max_idle_time) == SUCCESS) + { + g_connection_pool_max_idle_time = \ + atoi(conn_pool_max_idle_time.value.str.val); + if (g_connection_pool_max_idle_time <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "%s: %d in config filename" \ + "is invalid!", __LINE__, \ + ITEM_NAME_CONN_POOL_MAX_IDLE_TIME, \ + g_connection_pool_max_idle_time); + return EINVAL; + } + } + else + { + g_connection_pool_max_idle_time = 3600; + } + + g_use_connection_pool = true; + result = conn_pool_init(&g_connection_pool, \ + g_fdfs_connect_timeout, \ + 0, g_connection_pool_max_idle_time); + if (result != 0) + { + return result; + } + } + } + + + logDebug("base_path=%s, connect_timeout=%d, network_timeout=%d, " \ + "anti_steal_secret_key length=%d, " \ + "tracker_group_count=%d, first tracker group server_count=%d, "\ + "use_connection_pool=%d, connection_pool_max_idle_time: %d", \ + g_fdfs_base_path, g_fdfs_connect_timeout, \ + g_fdfs_network_timeout, (int)strlen(pAntiStealSecretKey), \ + config_count, g_tracker_group.server_count, \ + g_use_connection_pool, g_connection_pool_max_idle_time); + + return 0; +} + +PHP_MINIT_FUNCTION(fastdfs_client) +{ + zend_class_entry ce; + + log_init(); + if (load_config_files() != 0) + { + return FAILURE; + } + + le_fdht = zend_register_list_destructors_ex(NULL, php_fdfs_dtor, \ + "FastDFS", module_number); + + INIT_CLASS_ENTRY(ce, "FastDFS", fdfs_class_methods); + fdfs_ce = zend_register_internal_class(&ce TSRMLS_CC); + fdfs_ce->create_object = php_fdfs_new; + + INIT_CLASS_ENTRY(ce, "FastDFSException", NULL); + fdfs_exception_ce = zend_register_internal_class_ex(&ce, \ + php_fdfs_get_exception_base(0 TSRMLS_CC), NULL TSRMLS_CC); + + REGISTER_STRING_CONSTANT("FDFS_FILE_ID_SEPERATOR", \ + FDFS_FILE_ID_SEPERATE_STR, \ + CONST_CS | CONST_PERSISTENT); + + REGISTER_STRING_CONSTANT("FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE", \ + STORAGE_SET_METADATA_FLAG_OVERWRITE_STR, \ + CONST_CS | CONST_PERSISTENT); + + REGISTER_STRING_CONSTANT("FDFS_STORAGE_SET_METADATA_FLAG_MERGE", \ + STORAGE_SET_METADATA_FLAG_MERGE_STR, \ + CONST_CS | CONST_PERSISTENT); + + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_INIT", \ + FDFS_STORAGE_STATUS_INIT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_WAIT_SYNC", \ + FDFS_STORAGE_STATUS_WAIT_SYNC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_SYNCING", \ + FDFS_STORAGE_STATUS_SYNCING, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_DELETED", \ + FDFS_STORAGE_STATUS_DELETED, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_OFFLINE", \ + FDFS_STORAGE_STATUS_OFFLINE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_ONLINE", \ + FDFS_STORAGE_STATUS_ONLINE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_ACTIVE", \ + FDFS_STORAGE_STATUS_ACTIVE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_NONE", \ + FDFS_STORAGE_STATUS_NONE, CONST_CS | CONST_PERSISTENT); + + return SUCCESS; +} + +PHP_MSHUTDOWN_FUNCTION(fastdfs_client) +{ + FDFSConfigInfo *pConfigInfo; + FDFSConfigInfo *pConfigEnd; + + if (config_list != NULL) + { + pConfigEnd = config_list + config_count; + for (pConfigInfo=config_list; pConfigInfopTrackerGroup != NULL) + { + tracker_close_all_connections_ex( \ + pConfigInfo->pTrackerGroup); + } + } + } + + if (g_use_connection_pool) + { + fdfs_connection_pool_destroy(); + } + + fdfs_client_destroy(); + log_destroy(); + + return SUCCESS; +} + +PHP_RINIT_FUNCTION(fastdfs_client) +{ + return SUCCESS; +} + +PHP_RSHUTDOWN_FUNCTION(fastdfs_client) +{ + fprintf(stderr, "request shut down. file: "__FILE__", line: %d\n", __LINE__); + return SUCCESS; +} + +PHP_MINFO_FUNCTION(fastdfs_client) +{ + char fastdfs_info[64]; + sprintf(fastdfs_info, "fastdfs_client v%d.%02d support", + g_fdfs_version.major, g_fdfs_version.minor); + + php_info_print_table_start(); + php_info_print_table_header(2, fastdfs_info, "enabled"); + php_info_print_table_end(); +} + diff --git a/php_client/fastdfs_client.h b/php_client/fastdfs_client.h new file mode 100644 index 0000000..14b5c10 --- /dev/null +++ b/php_client/fastdfs_client.h @@ -0,0 +1,100 @@ +#ifndef FASTDFS_CLIENT_H +#define FASTDFS_CLIENT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef PHP_WIN32 +#define PHP_FASTDFS_API __declspec(dllexport) +#else +#define PHP_FASTDFS_API +#endif + +PHP_MINIT_FUNCTION(fastdfs_client); +PHP_RINIT_FUNCTION(fastdfs_client); +PHP_MSHUTDOWN_FUNCTION(fastdfs_client); +PHP_RSHUTDOWN_FUNCTION(fastdfs_client); +PHP_MINFO_FUNCTION(fastdfs_client); + +ZEND_FUNCTION(fastdfs_client_version); +ZEND_FUNCTION(fastdfs_active_test); +ZEND_FUNCTION(fastdfs_connect_server); +ZEND_FUNCTION(fastdfs_disconnect_server); +ZEND_FUNCTION(fastdfs_get_last_error_no); +ZEND_FUNCTION(fastdfs_get_last_error_info); + +ZEND_FUNCTION(fastdfs_tracker_get_connection); +ZEND_FUNCTION(fastdfs_tracker_make_all_connections); +ZEND_FUNCTION(fastdfs_tracker_close_all_connections); +ZEND_FUNCTION(fastdfs_tracker_list_groups); +ZEND_FUNCTION(fastdfs_tracker_query_storage_store); +ZEND_FUNCTION(fastdfs_tracker_query_storage_store_list); +ZEND_FUNCTION(fastdfs_tracker_query_storage_update); +ZEND_FUNCTION(fastdfs_tracker_query_storage_fetch); +ZEND_FUNCTION(fastdfs_tracker_query_storage_list); +ZEND_FUNCTION(fastdfs_tracker_query_storage_update1); +ZEND_FUNCTION(fastdfs_tracker_query_storage_fetch1); +ZEND_FUNCTION(fastdfs_tracker_query_storage_list1); +ZEND_FUNCTION(fastdfs_tracker_delete_storage); +ZEND_FUNCTION(fastdfs_storage_upload_by_filename); +ZEND_FUNCTION(fastdfs_storage_upload_by_filename1); +ZEND_FUNCTION(fastdfs_storage_upload_by_filebuff); +ZEND_FUNCTION(fastdfs_storage_upload_by_filebuff1); +ZEND_FUNCTION(fastdfs_storage_upload_by_callback); +ZEND_FUNCTION(fastdfs_storage_upload_by_callback1); +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filename); +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filename1); +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filebuff); +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_filebuff1); +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_callback); +ZEND_FUNCTION(fastdfs_storage_upload_appender_by_callback1); +ZEND_FUNCTION(fastdfs_storage_delete_file); +ZEND_FUNCTION(fastdfs_storage_delete_file1); +ZEND_FUNCTION(fastdfs_storage_download_file_to_buff); +ZEND_FUNCTION(fastdfs_storage_download_file_to_buff1); +ZEND_FUNCTION(fastdfs_storage_download_file_to_file); +ZEND_FUNCTION(fastdfs_storage_download_file_to_file1); +ZEND_FUNCTION(fastdfs_storage_download_file_to_callback); +ZEND_FUNCTION(fastdfs_storage_download_file_to_callback1); +ZEND_FUNCTION(fastdfs_storage_set_metadata); +ZEND_FUNCTION(fastdfs_storage_set_metadata1); +ZEND_FUNCTION(fastdfs_storage_get_metadata); +ZEND_FUNCTION(fastdfs_storage_get_metadata1); +ZEND_FUNCTION(fastdfs_http_gen_token); +ZEND_FUNCTION(fastdfs_get_file_info); +ZEND_FUNCTION(fastdfs_get_file_info1); +ZEND_FUNCTION(fastdfs_storage_file_exist); +ZEND_FUNCTION(fastdfs_storage_file_exist1); +ZEND_FUNCTION(fastdfs_gen_slave_filename); +ZEND_FUNCTION(fastdfs_send_data); +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filename); +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filename1); +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filebuff); +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_filebuff1); +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_callback); +ZEND_FUNCTION(fastdfs_storage_upload_slave_by_callback1); +ZEND_FUNCTION(fastdfs_storage_append_by_filename); +ZEND_FUNCTION(fastdfs_storage_append_by_filename1); +ZEND_FUNCTION(fastdfs_storage_append_by_filebuff); +ZEND_FUNCTION(fastdfs_storage_append_by_filebuff1); +ZEND_FUNCTION(fastdfs_storage_append_by_callback); +ZEND_FUNCTION(fastdfs_storage_append_by_callback1); +ZEND_FUNCTION(fastdfs_storage_modify_by_filename); +ZEND_FUNCTION(fastdfs_storage_modify_by_filename1); +ZEND_FUNCTION(fastdfs_storage_modify_by_filebuff); +ZEND_FUNCTION(fastdfs_storage_modify_by_filebuff1); +ZEND_FUNCTION(fastdfs_storage_modify_by_callback); +ZEND_FUNCTION(fastdfs_storage_modify_by_callback1); +ZEND_FUNCTION(fastdfs_storage_truncate_file); +ZEND_FUNCTION(fastdfs_storage_truncate_file1); + +PHP_FASTDFS_API zend_class_entry *php_fdfs_get_ce(void); +PHP_FASTDFS_API zend_class_entry *php_fdfs_get_exception(void); +PHP_FASTDFS_API zend_class_entry *php_fdfs_get_exception_base(int root TSRMLS_DC); + +#ifdef __cplusplus +} +#endif + +#endif /* FASTDFS_CLIENT_H */ diff --git a/php_client/fastdfs_client.ini b/php_client/fastdfs_client.ini new file mode 100644 index 0000000..20f5db1 --- /dev/null +++ b/php_client/fastdfs_client.ini @@ -0,0 +1,52 @@ +extension = fastdfs_client.so + +; the base path +fastdfs_client.base_path = /tmp + +; connect timeout in seconds +; default value is 30s +fastdfs_client.connect_timeout = 2 + +; network timeout in seconds +; default value is 30s +fastdfs_client.network_timeout = 60 + +; standard log level as syslog, case insensitive, value list: +;;; emerg for emergency +;;; alert +;;; crit for critical +;;; error +;;; warn for warning +;;; notice +;;; info +;;; debug +fastdfs_client.log_level = info + +; set the log filename, such as /usr/local/fastdfs/logs/fastdfs_client.log +; empty for output to stderr +fastdfs_client.log_filename = + +; secret key to generate anti-steal token +; this parameter must be set when http.anti_steal.check_token set to true +; the length of the secret key should not exceed 128 bytes +fastdfs_client.http.anti_steal_secret_key = + +; FastDFS cluster count, default value is 1 +fastdfs_client.tracker_group_count = 1 + +; config file of FastDFS cluster ;, based 0 +; must include absolute path, such as fastdfs_client.tracker_group0 +; the config file is same as conf/client.conf +fastdfs_client.tracker_group0 = /etc/fdfs/client.conf + +; if use connection pool +; default value is false +; since V4.05 +fastdfs_client.use_connection_pool = false + +; connections whose the idle time exceeds this time will be closed +; unit: second +; default value is 3600 +; since V4.05 +fastdfs_client.connection_pool_max_idle_time = 3600 + diff --git a/php_client/fastdfs_test.php b/php_client/fastdfs_test.php new file mode 100644 index 0000000..8c74f33 --- /dev/null +++ b/php_client/fastdfs_test.php @@ -0,0 +1,398 @@ +1024, 'height'=>800, 'font'=>'Aris', 'Homepage' => true, 'price' => 103.75, 'status' => FDFS_STORAGE_STATUS_ACTIVE), '', $tracker, $storage); + if ($file_id) + { + $master_file_id = $file_id; + $prefix_name = '.part2'; + $slave_file_id = fastdfs_storage_upload_slave_by_filename1("/usr/include/stdio.h", + $master_file_id, $prefix_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = fastdfs_gen_slave_filename($master_file_id, $prefix_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . fastdfs_storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filename1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . fastdfs_storage_delete_file1($file_id) . "\n"; + } + + $file_info = fastdfs_storage_upload_by_filebuff("this is a test.", "txt"); + if ($file_info) + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + var_dump(fastdfs_get_file_info($group_name, $remote_filename)); + echo "file exist: " . fastdfs_storage_file_exist($group_name, $remote_filename) . "\n"; + + $ts = time(); + $token = fastdfs_http_gen_token($remote_filename, $ts); + echo "token=$token\n"; + + $file_content = fastdfs_storage_download_file_to_buff($file_info['group_name'], $file_info['filename']); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't1.txt'; + echo 'storage_download_file_to_file result: ' . + fastdfs_storage_download_file_to_file($file_info['group_name'], $file_info['filename'], $local_filename) . "\n"; + + echo "fastdfs_storage_set_metadata result: " . fastdfs_storage_set_metadata( + $file_info['group_name'], $file_info['filename'], + array('color'=>'', 'size'=>32, 'font'=>'MS Serif'), FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE) . "\n"; + + $meta_list = fastdfs_storage_get_metadata($file_info['group_name'], $file_info['filename']); + var_dump($meta_list); + + $master_filename = $remote_filename; + $prefix_name = '.part1'; + $file_ext_name = 'txt'; + $slave_file_info = fastdfs_storage_upload_slave_by_filebuff('this is slave file.', + $group_name, $master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = fastdfs_gen_slave_filename($master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . fastdfs_storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filebuff fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file return: " . fastdfs_storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_id = fastdfs_storage_upload_by_filebuff1("this\000is\000a\000test.", "bin", + array('width'=>1024, 'height'=>768, 'font'=>'Aris')); + if ($file_id) + { + $file_content = fastdfs_storage_download_file_to_buff1($file_id); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't2.txt'; + echo 'storage_download_file_to_file1 result: ' . + fastdfs_storage_download_file_to_file1($file_id, $local_filename) . "\n"; + echo "fastdfs_storage_set_metadata1 result: " . fastdfs_storage_set_metadata1( + $file_id, array('color'=>'yellow', 'size'=>'1234567890', 'font'=>'MS Serif'), + FDFS_STORAGE_SET_METADATA_FLAG_MERGE) . "\n"; + $meta_list = fastdfs_storage_get_metadata1($file_id); + var_dump($meta_list); + + $master_file_id = $file_id; + $prefix_name = '.part2'; + $file_ext_name = 'txt'; + $slave_file_id = fastdfs_storage_upload_slave_by_filebuff1('this is slave file1.', + $master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = fastdfs_gen_slave_filename($master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . fastdfs_storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filebuff1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . fastdfs_storage_delete_file1($file_id) . "\n"; + } + + var_dump(fastdfs_tracker_query_storage_update($group_name, $remote_filename)); + var_dump(fastdfs_tracker_query_storage_fetch($group_name, $remote_filename)); + var_dump(fastdfs_tracker_query_storage_list($group_name, $remote_filename)); + var_dump(fastdfs_tracker_query_storage_update1($file_id)); + var_dump(fastdfs_tracker_query_storage_fetch1($file_id, $tracker)); + var_dump(fastdfs_tracker_query_storage_list1($file_id, $tracker)); + + echo "fastdfs_tracker_close_all_connections result: " . fastdfs_tracker_close_all_connections() . "\n"; + + $fdfs = new FastDFS(); + echo 'tracker_make_all_connections result: ' . $fdfs->tracker_make_all_connections() . "\n"; + $tracker = $fdfs->tracker_get_connection(); + var_dump($tracker); + + $server = $fdfs->connect_server($tracker['ip_addr'], $tracker['port']); + var_dump($server); + var_dump($fdfs->disconnect_server($server)); + + var_dump($fdfs->tracker_query_storage_store_list()); + + //var_dump($fdfs->tracker_list_groups()); + //var_dump($fdfs->tracker_query_storage_store()); + //var_dump($fdfs->tracker_query_storage_update($group_name, $remote_filename)); + //var_dump($fdfs->tracker_query_storage_fetch($group_name, $remote_filename)); + //var_dump($fdfs->tracker_query_storage_list($group_name, $remote_filename)); + + var_dump($fdfs->tracker_query_storage_update1($file_id)); + var_dump($fdfs->tracker_query_storage_fetch1($file_id)); + var_dump($fdfs->tracker_query_storage_list1($file_id)); + + $file_info = $fdfs->storage_upload_by_filename("/usr/include/stdio.h"); + if ($file_info) + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + var_dump($fdfs->get_file_info($group_name, $remote_filename)); + echo "file exist: " . $fdfs->storage_file_exist($group_name, $remote_filename) . "\n"; + + $master_filename = $remote_filename; + $prefix_name = '.part1'; + $slave_file_info = $fdfs->storage_upload_slave_by_filename("/usr/include/stdio.h", + $group_name, $master_filename, $prefix_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = $fdfs->gen_slave_filename($master_filename, $prefix_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . $fdfs->storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "storage_upload_slave_by_filename fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file return: " . $fdfs->storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_ext_name = 'c'; + $file_id = $fdfs->storage_upload_by_filename1("/usr/include/stdio.h", $file_ext_name, array('width'=>1024, 'height'=>800, 'font'=>'Aris')); + if ($file_id) + { + $master_file_id = $file_id; + $prefix_name = '.part2'; + $slave_file_id = $fdfs->storage_upload_slave_by_filename1("/usr/include/stdio.h", + $master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = $fdfs->gen_slave_filename($master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . $fdfs->storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . $fdfs->storage_delete_file1($file_id) . "\n"; + } + + $file_info = $fdfs->storage_upload_by_filebuff("", "txt"); + if ($file_info) + { + var_dump($file_info); + $file_content = $fdfs->storage_download_file_to_buff($file_info['group_name'], $file_info['filename']); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't3.txt'; + echo 'storage_download_file_to_file result: ' . + $fdfs->storage_download_file_to_file($file_info['group_name'], $file_info['filename'], $local_filename) . "\n"; + + echo "storage_set_metadata result: " . $fdfs->storage_set_metadata( + $file_info['group_name'], $file_info['filename'], + array('color'=>'yellow', 'size'=>32), FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE) . "\n"; + + $meta_list = $fdfs->storage_get_metadata($file_info['group_name'], $file_info['filename']); + var_dump($meta_list); + + $master_filename = $file_info['filename']; + $prefix_name = '.part1'; + $file_ext_name = 'txt'; + $slave_file_info = $fdfs->storage_upload_slave_by_filebuff('this is slave file 1 by class.', + $file_info['group_name'], $master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = $fdfs->gen_slave_filename($master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . $fdfs->storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "storage_upload_slave_by_filebuff fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file return: " . $fdfs->storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_id = $fdfs->storage_upload_by_filebuff1("this\000is\001a\002test.", "bin", + array('color'=>'none', 'size'=>0, 'font'=>'Aris')); + if ($file_id) + { + var_dump($fdfs->get_file_info1($file_id)); + echo "file exist: " . $fdfs->storage_file_exist1($file_id) . "\n"; + + $ts = time(); + $token = $fdfs->http_gen_token($file_id, $ts); + echo "token=$token\n"; + + $file_content = $fdfs->storage_download_file_to_buff1($file_id); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't4.txt'; + echo 'storage_download_file_to_file1 result: ' . $fdfs->storage_download_file_to_file1($file_id, $local_filename) . "\n"; + echo "storage_set_metadata1 result: " . $fdfs->storage_set_metadata1( + $file_id, array('color'=>'yellow', 'size'=>32), FDFS_STORAGE_SET_METADATA_FLAG_MERGE) . "\n"; + + $master_file_id = $file_id; + $prefix_name = '.part2'; + $file_ext_name = 'txt'; + $slave_file_id = $fdfs->storage_upload_slave_by_filebuff1('this is slave file 2 by class.', + $master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = $fdfs->gen_slave_filename($master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . $fdfs->storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "storage_upload_slave_by_filebuff1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + $meta_list = $fdfs->storage_get_metadata1($file_id); + if ($meta_list !== false) + { + var_dump($meta_list); + } + else + { + echo "errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . $fdfs->storage_delete_file1($file_id) . "\n"; + } + + var_dump($fdfs->active_test($tracker)); + echo 'tracker_close_all_connections result: ' . $fdfs->tracker_close_all_connections() . "\n"; +?> diff --git a/php_client/fastdfs_test1.php b/php_client/fastdfs_test1.php new file mode 100644 index 0000000..9030f4e --- /dev/null +++ b/php_client/fastdfs_test1.php @@ -0,0 +1,446 @@ +1024, 'heigth'=>768, 'color'=>'red'); + $file_info = fastdfs_storage_upload_slave_by_filebuff("/usr/include/stdio.h", 'group1', 'M00/01/14/wKgAxU5n9gUIAAAAAAAD8uiojuUAAAAEgP_8YAAAAQK66492', '_100x100', 'h', $meta_list); + if ($file_info) + { + var_dump($file_info); + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + } + else + { + $group_name = ''; + $remote_filename = ''; + } + + //$result = fastdfs_storage_set_metadata1($group_name, $remote_filename, + // $meta_list, FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE); + // echo "fastdfs_storage_set_metadata result: $result\n"; + + var_dump(fastdfs_storage_get_metadata($group_name, $remote_filename)); + error_log("errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info()); + */ + + /* + $file_id = $group_name . FDFS_FILE_ID_SEPERATOR . 'M00/00/02/wKjRbExc_qIAAAAAAABtNw6hsnM56585.part2.c'; + + var_dump(fastdfs_get_file_info1($file_id)); + exit(1); + */ + + echo 'fastdfs_tracker_make_all_connections result: ' . fastdfs_tracker_make_all_connections() . "\n"; + var_dump(fastdfs_tracker_list_groups()); + + $tracker = fastdfs_tracker_get_connection(); + var_dump($tracker); + + if (!fastdfs_active_test($tracker)) + { + error_log("errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info()); + exit(1); + } + + $server = fastdfs_connect_server($tracker['ip_addr'], $tracker['port']); + var_dump($server); + var_dump(fastdfs_disconnect_server($server)); + var_dump($server); + + var_dump(fastdfs_tracker_query_storage_store_list()); + + $storage = fastdfs_tracker_query_storage_store(); + if (!$storage) + { + error_log("errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info()); + exit(1); + } + + $server = fastdfs_connect_server($storage['ip_addr'], $storage['port']); + if (!$server) + { + error_log("errno1: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info()); + exit(1); + } + if (!fastdfs_active_test($server)) + { + error_log("errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info()); + exit(1); + } + + //var_dump(fastdfs_tracker_list_groups($tracker)); + $storage['sock'] = $server['sock']; + $file_info = fastdfs_storage_upload_by_filename("/usr/include/stdio.h", null, array(), null, $tracker, $storage); + if (!$file_info) + { + echo "fastdfs_storage_upload_by_filename fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + $finfo = fastdfs_get_file_info($group_name, $remote_filename); + if (!$finfo) + { + echo "fastdfs_get_file_info fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + var_dump($finfo); + } + + $master_filename = $remote_filename; + $prefix_name = '.part1'; + $slave_file_info = fastdfs_storage_upload_slave_by_filename("/usr/include/stdio.h", + $group_name, $master_filename, $prefix_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = fastdfs_gen_slave_filename($master_filename, $prefix_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . fastdfs_storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filename fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file return: " . fastdfs_storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_id = fastdfs_storage_upload_by_filename1("/usr/include/stdio.h", null, array('width'=>1024, 'height'=>800, 'font'=>'Aris', 'Homepage' => true, 'price' => 103.75, 'status' => FDFS_STORAGE_STATUS_ACTIVE), '', $tracker, $storage); + fastdfs_disconnect_server($storage); + if (!$file_id) + { + echo "fastdfs_storage_upload_by_filename1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + $master_file_id = $file_id; + $prefix_name = '.part2'; + $slave_file_id = fastdfs_storage_upload_slave_by_filename1("/usr/include/stdio.h", + $master_file_id, $prefix_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = fastdfs_gen_slave_filename($master_file_id, $prefix_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . fastdfs_storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filename1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . fastdfs_storage_delete_file1($file_id) . "\n"; + } + + $file_info = fastdfs_storage_upload_by_filebuff("this is a test.", "txt"); + if (!$file_info) + { + echo "fastdfs_storage_upload_by_filebuff fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + var_dump($file_info); + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump(fastdfs_get_file_info($group_name, $remote_filename)); + + $ts = time(); + $token = fastdfs_http_gen_token($remote_filename, $ts); + echo "token=$token\n"; + + $file_content = fastdfs_storage_download_file_to_buff($file_info['group_name'], $file_info['filename']); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't1.txt'; + echo 'storage_download_file_to_file result: ' . + fastdfs_storage_download_file_to_file($file_info['group_name'], $file_info['filename'], $local_filename) . "\n"; + + echo "fastdfs_storage_set_metadata result: " . fastdfs_storage_set_metadata( + $file_info['group_name'], $file_info['filename'], + array('color'=>'', 'size'=>32, 'font'=>'MS Serif'), FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE) . "\n"; + + $meta_list = fastdfs_storage_get_metadata($file_info['group_name'], $file_info['filename']); + var_dump($meta_list); + + $master_filename = $remote_filename; + $prefix_name = '.part1'; + $file_ext_name = 'txt'; + $slave_file_info = fastdfs_storage_upload_slave_by_filebuff('this is slave file.', + $group_name, $master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = fastdfs_gen_slave_filename($master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . fastdfs_storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filebuff fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file return: " . fastdfs_storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_id = fastdfs_storage_upload_by_filebuff1("this\000is\000a\000test.", "bin", + array('width'=>1024, 'height'=>768, 'font'=>'Aris')); + if (!$file_id) + { + echo "fastdfs_storage_upload_by_filebuff1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + $file_content = fastdfs_storage_download_file_to_buff1($file_id); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't2.txt'; + echo 'storage_download_file_to_file1 result: ' . + fastdfs_storage_download_file_to_file1($file_id, $local_filename) . "\n"; + echo "fastdfs_storage_set_metadata1 result: " . fastdfs_storage_set_metadata1( + $file_id, array('color'=>'yellow', 'size'=>'1234567890', 'font'=>'MS Serif'), + FDFS_STORAGE_SET_METADATA_FLAG_MERGE) . "\n"; + $meta_list = fastdfs_storage_get_metadata1($file_id); + var_dump($meta_list); + + $master_file_id = $file_id; + $prefix_name = '.part2'; + $file_ext_name = 'txt'; + $slave_file_id = fastdfs_storage_upload_slave_by_filebuff1('this is slave file1.', + $master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = fastdfs_gen_slave_filename($master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . fastdfs_storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filebuff1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . fastdfs_storage_delete_file1($file_id) . "\n"; + } + + + var_dump(fastdfs_tracker_query_storage_update($group_name, $remote_filename)); + var_dump(fastdfs_tracker_query_storage_fetch($group_name, $remote_filename)); + var_dump(fastdfs_tracker_query_storage_list($group_name, $remote_filename)); + var_dump(fastdfs_tracker_query_storage_update1($file_id)); + var_dump(fastdfs_tracker_query_storage_fetch1($file_id, $tracker)); + var_dump(fastdfs_tracker_query_storage_list1($file_id, $tracker)); + + echo "fastdfs_tracker_close_all_connections result: " . fastdfs_tracker_close_all_connections() . "\n"; + $fdfs = new FastDFS(); + echo 'tracker_make_all_connections result: ' . $fdfs->tracker_make_all_connections() . "\n"; + $tracker = $fdfs->tracker_get_connection(); + var_dump($tracker); + + $server = $fdfs->connect_server($tracker['ip_addr'], $tracker['port']); + var_dump($server); + var_dump($fdfs->disconnect_server($server)); + + var_dump($fdfs->tracker_query_storage_store_list()); + + //var_dump($fdfs->tracker_list_groups()); + //var_dump($fdfs->tracker_query_storage_store()); + //var_dump($fdfs->tracker_query_storage_update($group_name, $remote_filename)); + //var_dump($fdfs->tracker_query_storage_fetch($group_name, $remote_filename)); + //var_dump($fdfs->tracker_query_storage_list($group_name, $remote_filename)); + + var_dump($fdfs->tracker_query_storage_update1($file_id)); + var_dump($fdfs->tracker_query_storage_fetch1($file_id)); + var_dump($fdfs->tracker_query_storage_list1($file_id)); + + $file_info = $fdfs->storage_upload_by_filename("/usr/include/stdio.h"); + if (!$file_info) + { + echo "storage_upload_by_filename fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + var_dump($fdfs->get_file_info($group_name, $remote_filename)); + + $master_filename = $remote_filename; + $prefix_name = '.part1'; + $slave_file_info = $fdfs->storage_upload_slave_by_filename("/usr/include/stdio.h", + $group_name, $master_filename, $prefix_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = $fdfs->gen_slave_filename($master_filename, $prefix_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . $fdfs->storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "storage_upload_slave_by_filename fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file return: " . $fdfs->storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_ext_name = 'c'; + $file_id = $fdfs->storage_upload_by_filename1("/usr/include/stdio.h", $file_ext_name, array('width'=>1024, 'height'=>800, 'font'=>'Aris')); + if (!$file_id) + { + echo "storage_upload_by_filename1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + else + { + $master_file_id = $file_id; + $prefix_name = '.part2'; + $slave_file_id = $fdfs->storage_upload_slave_by_filename1("/usr/include/stdio.h", + $master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = $fdfs->gen_slave_filename($master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + $result = $fdfs->storage_delete_file1($slave_file_id); + echo "delete file $slave_file_id return: $result\n"; + } + else + { + echo "storage_upload_slave_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . $fdfs->storage_delete_file1($file_id) . "\n"; + } + + $file_info = $fdfs->storage_upload_by_filebuff("", "txt"); + if ($file_info) + { + var_dump($file_info); + $file_content = $fdfs->storage_download_file_to_buff($file_info['group_name'], $file_info['filename']); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't3.txt'; + echo 'storage_download_file_to_file result: ' . + $fdfs->storage_download_file_to_file($file_info['group_name'], $file_info['filename'], $local_filename) . "\n"; + + echo "storage_set_metadata result: " . $fdfs->storage_set_metadata( + $file_info['group_name'], $file_info['filename'], + array('color'=>'yellow', 'size'=>32), FDFS_STORAGE_SET_METADATA_FLAG_OVERWRITE) . "\n"; + + $meta_list = $fdfs->storage_get_metadata($file_info['group_name'], $file_info['filename']); + var_dump($meta_list); + + $master_filename = $file_info['filename']; + $prefix_name = '.part1'; + $file_ext_name = 'txt'; + $slave_file_info = $fdfs->storage_upload_slave_by_filebuff('this is slave file 1 by class.', + $file_info['group_name'], $master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = $fdfs->gen_slave_filename($master_filename, $prefix_name, $file_ext_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + echo "delete slave file return: " . $fdfs->storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "storage_upload_slave_by_filebuff fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file return: " . $fdfs->storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + + $file_id = $fdfs->storage_upload_by_filebuff1("this\000is\001a\002test.", "bin", + array('color'=>'none', 'size'=>0, 'font'=>'Aris')); + if ($file_id) + { + var_dump($fdfs->get_file_info1($file_id)); + + $ts = time(); + $token = $fdfs->http_gen_token($file_id, $ts); + echo "token=$token\n"; + + $file_content = $fdfs->storage_download_file_to_buff1($file_id); + echo "file content: " . $file_content . "(" . strlen($file_content) . ")\n"; + $local_filename = 't4.txt'; + echo 'storage_download_file_to_file1 result: ' . $fdfs->storage_download_file_to_file1($file_id, $local_filename) . "\n"; + echo "storage_set_metadata1 result: " . $fdfs->storage_set_metadata1( + $file_id, array('color'=>'yellow', 'size'=>32), FDFS_STORAGE_SET_METADATA_FLAG_MERGE) . "\n"; + + $master_file_id = $file_id; + $prefix_name = '.part2'; + $file_ext_name = 'txt'; + $slave_file_id = $fdfs->storage_upload_slave_by_filebuff1('this is slave file 2 by class.', + $master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id !== false) + { + var_dump($slave_file_id); + + $generated_file_id = $fdfs->gen_slave_filename($master_file_id, $prefix_name, $file_ext_name); + if ($slave_file_id != $generated_file_id) + { + echo "${slave_file_id}\n != \n${generated_file_id}\n"; + } + + echo "delete file $slave_file_id return: " . $fdfs->storage_delete_file1($slave_file_id) . "\n"; + } + else + { + echo "storage_upload_slave_by_filebuff1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + $meta_list = $fdfs->storage_get_metadata1($file_id); + if ($meta_list !== false) + { + var_dump($meta_list); + } + else + { + echo "errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n"; + } + + echo "delete file $file_id return: " . $fdfs->storage_delete_file1($file_id) . "\n"; + } + + var_dump($fdfs->active_test($tracker)); + echo 'tracker_close_all_connections result: ' . $fdfs->tracker_close_all_connections() . "\n"; +?> diff --git a/php_client/fastdfs_test_slave.php b/php_client/fastdfs_test_slave.php new file mode 100644 index 0000000..fb70d50 --- /dev/null +++ b/php_client/fastdfs_test_slave.php @@ -0,0 +1,62 @@ + 1)); + if ($file_info) + { + $group_name = $file_info['group_name']; + $remote_filename = $file_info['filename']; + + var_dump($file_info); + var_dump(fastdfs_get_file_info($group_name, $remote_filename)); + + $master_filename = $remote_filename; + $prefix_name = '.part1'; + $meta_list = array('width' => 1024, 'height' => 768, 'color' => 'blue'); + $slave_file_info = fastdfs_storage_upload_slave_by_filename("/usr/include/stdio.h", + $group_name, $master_filename, $prefix_name, null, $meta_list); + if ($slave_file_info !== false) + { + var_dump($slave_file_info); + + $generated_filename = fastdfs_gen_slave_filename($master_filename, $prefix_name); + if ($slave_file_info['filename'] != $generated_filename) + { + echo "${slave_file_info['filename']}\n != \n${generated_filename}\n"; + } + + //echo "delete slave file return: " . fastdfs_storage_delete_file($slave_file_info['group_name'], $slave_file_info['filename']) . "\n"; + } + else + { + echo "fastdfs_storage_upload_slave_by_filename fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n"; + } + + echo "delete file return: " . fastdfs_storage_delete_file($file_info['group_name'], $file_info['filename']) . "\n"; + } + +?> diff --git a/restart.sh b/restart.sh new file mode 100755 index 0000000..2b70250 --- /dev/null +++ b/restart.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +if [ -z "$1" ]; then + /bin/echo "$0 " + exit 1 +fi + +if [ -f /bin/awk ]; then + AWK=/bin/awk +else + AWK=/usr/bin/awk +fi + +if [ -f /bin/grep ]; then + GREP=/bin/grep +else + GREP=/usr/bin/grep +fi + +if [ -f /bin/expr ]; then + EXPR=/bin/expr +else + EXPR=/usr/bin/expr +fi + +if [ -f /bin/sed ]; then + SED=/bin/sed +else + SED=/usr/bin/sed +fi + +program=`/bin/echo $1 | $AWK -F '/' '{print $NF;}'` +param='' +grep_cmd="$GREP -w $program" + +list='2 3 4 5 6 7 8 9' +for i in $list; do + eval p='$'$i + if [ -z "$p" ]; then + break + fi + param="$param $p" + #first_ch=`$EXPR substr "$p" 1 1` + first_ch=`/bin/echo "$p" | $SED -e 's/\(.\).*/\1/'` + if [ "$first_ch" = "-" ]; then + p="'\\$p'" + fi + grep_cmd="$grep_cmd | $GREP -w $p" +done + +cmd="/bin/ps auxww | $grep_cmd | $GREP -v grep | $GREP -v $0 | $AWK '{print \$2;}'" +pids=`/bin/sh -c "$cmd"` +if [ ! -z "$pids" ]; then + i=0 + count=0 + /bin/echo "stopping $program ..." + while [ 1 -eq 1 ]; do + new_pids='' + for pid in $pids; do + if [ $i -eq 0 ]; then + /bin/kill $pid + else + /bin/kill $pid >/dev/null 2>&1 + fi + + if [ $? -eq 0 ]; then + new_pids="$new_pids $pid" + fi + count=`$EXPR $count + 1` + done + + if [ -z "$new_pids" ]; then + break + fi + + pids="$new_pids" + /usr/bin/printf . + /bin/sleep 1 + i=`$EXPR $i + 1` + done +fi + +/bin/echo "" +cmd="/bin/ps auxww | $grep_cmd | $GREP -v grep | $GREP -v $0 | /usr/bin/wc -l" +count=`/bin/sh -c "$cmd"` +if [ $count -eq 0 ]; then + /bin/echo "starting $program ..." + exec $1 $param + exit $? +else + cmd="/bin/ps auxww | $grep_cmd | $GREP -v grep | $GREP -v $0" + /bin/sh -c "$cmd" + /bin/echo "already running $program count: $count, restart aborted!" + exit 16 +fi + diff --git a/stop.sh b/stop.sh new file mode 100755 index 0000000..6cbdab9 --- /dev/null +++ b/stop.sh @@ -0,0 +1,92 @@ +#!/bin/sh + +if [ -z "$1" ]; then + /bin/echo "$0 " + exit 1 +fi + +if [ -f /bin/awk ]; then + AWK=/bin/awk +else + AWK=/usr/bin/awk +fi + +if [ -f /bin/grep ]; then + GREP=/bin/grep +else + GREP=/usr/bin/grep +fi + +if [ -f /bin/expr ]; then + EXPR=/bin/expr +else + EXPR=/usr/bin/expr +fi + +if [ -f /bin/sed ]; then + SED=/bin/sed +else + SED=/usr/bin/sed +fi + +program=`/bin/echo $1 | $AWK -F '/' '{print $NF;}'` +grep_cmd="$GREP -w $program" + +list='2 3 4 5 6 7 8 9' +for i in $list; do + eval p='$'$i + if [ -z "$p" ]; then + break + fi + #first_ch=`$EXPR substr "$p" 1 1` + first_ch=`/bin/echo "$p" | $SED -e 's/\(.\).*/\1/'` + if [ "$first_ch" = "-" ]; then + p="'\\$p'" + fi + grep_cmd="$grep_cmd | $GREP -w $p" +done + +cmd="/bin/ps auxww | $grep_cmd | $GREP -v grep | $GREP -v $0 | $AWK '{print \$2;}'" +pids=`/bin/sh -c "$cmd"` +if [ ! -z "$pids" ]; then + i=0 + count=0 + /bin/echo "stopping $program ..." + while [ 1 -eq 1 ]; do + new_pids='' + for pid in $pids; do + if [ $i -eq 0 ]; then + /bin/kill $pid + else + /bin/kill $pid >/dev/null 2>&1 + fi + + if [ $? -eq 0 ]; then + new_pids="$new_pids $pid" + fi + count=`$EXPR $count + 1` + done + + if [ -z "$new_pids" ]; then + break + fi + + pids="$new_pids" + /usr/bin/printf . + /bin/sleep 1 + i=`$EXPR $i + 1` + done +fi + +/bin/echo "" +cmd="/bin/ps auxww | $grep_cmd | $GREP -v grep | $GREP -v $0 | /usr/bin/wc -l" +count=`/bin/sh -c "$cmd"` +if [ $count -eq 0 ]; then + exit 0 +else + cmd="/bin/ps auxww | $grep_cmd | $GREP -v grep | $GREP -v $0" + /bin/sh -c "$cmd" + /bin/echo "already running $program count: $count, stop fail!" + exit 16 +fi + diff --git a/storage/Makefile.in b/storage/Makefile.in new file mode 100644 index 0000000..4ebc2cf --- /dev/null +++ b/storage/Makefile.in @@ -0,0 +1,49 @@ +.SUFFIXES: .c .o + +COMPILE = $(CC) $(CFLAGS) +INC_PATH = -I. -Itrunk_mgr -I../common -I../tracker -I../client -Ifdht_client -I/usr/local/include +LIB_PATH = -L/usr/local/lib $(LIBS) +TARGET_PATH = $(TARGET_PREFIX)/bin +CONFIG_PATH = $(TARGET_CONF_PATH) + +SHARED_OBJS = ../common/hash.o ../common/chain.o \ + ../common/shared_func.o ../common/ini_file_reader.o \ + ../common/logger.o ../common/sockopt.o ../common/fdfs_global.o \ + ../common/base64.o ../common/sched_thread.o \ + ../common/local_ip_func.o ../common/http_func.o \ + ../common/md5.o ../common/pthread_func.o \ + ../common/fast_mblock.o ../common/avl_tree.o \ + ../common/connection_pool.o ../common/process_ctrl.o \ + ../common/ioevent.o ../common/fast_timer.o \ + ../common/fast_task_queue.o ../common/ioevent_loop.o \ + ../tracker/fdfs_shared_func.o ../tracker/tracker_proto.o \ + tracker_client_thread.o storage_global.o storage_func.o \ + storage_service.o storage_sync.o storage_nio.o storage_dio.o \ + storage_ip_changed_dealer.o storage_param_getter.o \ + storage_disk_recovery.o trunk_mgr/trunk_mem.o \ + trunk_mgr/trunk_shared.o trunk_mgr/trunk_sync.o \ + trunk_mgr/trunk_client.o trunk_mgr/trunk_free_block_checker.o \ + ../client/client_global.o ../client/tracker_client.o \ + ../client/storage_client.o ../client/client_func.o \ + fdht_client/fdht_proto.o fdht_client/fdht_client.o \ + fdht_client/fdht_func.o fdht_client/fdht_global.o \ + $(STORAGE_EXTRA_OBJS) + +ALL_OBJS = $(SHARED_OBJS) + +ALL_PRGS = fdfs_storaged + +all: $(ALL_OBJS) $(ALL_PRGS) +.o: + $(COMPILE) -o $@ $< $(SHARED_OBJS) $(LIB_PATH) $(INC_PATH) +.c: + $(COMPILE) -o $@ $< $(ALL_OBJS) $(LIB_PATH) $(INC_PATH) +.c.o: + $(COMPILE) -c -o $@ $< $(INC_PATH) +install: + mkdir -p $(TARGET_PATH) + mkdir -p $(CONFIG_PATH) + cp -f $(ALL_PRGS) $(TARGET_PATH) + if [ ! -f $(CONFIG_PATH)/storage.conf ]; then cp -f ../conf/storage.conf ../conf/mime.types ../conf/http.conf $(CONFIG_PATH); fi +clean: + rm -f $(ALL_OBJS) $(ALL_PRGS) diff --git a/storage/fdfs_storaged.c b/storage/fdfs_storaged.c new file mode 100644 index 0000000..da508e8 --- /dev/null +++ b/storage/fdfs_storaged.c @@ -0,0 +1,602 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "pthread_func.h" +#include "process_ctrl.h" +#include "logger.h" +#include "fdfs_global.h" +#include "ini_file_reader.h" +#include "sockopt.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_client_thread.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_sync.h" +#include "storage_service.h" +#include "sched_thread.h" +#include "storage_dio.h" +#include "trunk_mem.h" +#include "trunk_sync.h" +#include "trunk_shared.h" + +#ifdef WITH_HTTPD +#include "storage_httpd.h" +#endif + +#if defined(DEBUG_FLAG) + +/* +#if defined(OS_LINUX) +#include "linux_stack_trace.h" +static bool bSegmentFault = false; +#endif +*/ + +#include "storage_dump.h" +#endif + +static bool bTerminateFlag = false; +static bool bAcceptEndFlag = false; + +static void sigQuitHandler(int sig); +static void sigHupHandler(int sig); +static void sigUsrHandler(int sig); +static void sigAlarmHandler(int sig); + +#if defined(DEBUG_FLAG) + +/* +#if defined(OS_LINUX) +static void sigSegvHandler(int signum, siginfo_t *info, void *ptr); +#endif +*/ + +static void sigDumpHandler(int sig); +#endif + +#define SCHEDULE_ENTRIES_MAX_COUNT 7 + +static void usage(const char *program) +{ + fprintf(stderr, "Usage: %s [start | stop | restart]\n", + program); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + int result; + int sock; + int wait_count; + pthread_t schedule_tid; + struct sigaction act; + ScheduleEntry scheduleEntries[SCHEDULE_ENTRIES_MAX_COUNT]; + ScheduleArray scheduleArray; + char pidFilename[MAX_PATH_SIZE]; + bool stop; + + if (argc < 2) + { + usage(argv[0]); + return 1; + } + + g_current_time = time(NULL); + g_up_time = g_current_time; + + log_init(); + trunk_shared_init(); + + conf_filename = argv[1]; + if ((result=get_base_path_from_conf_file(conf_filename, + g_fdfs_base_path, sizeof(g_fdfs_base_path))) != 0) + { + log_destroy(); + return result; + } + + snprintf(pidFilename, sizeof(pidFilename), + "%s/data/fdfs_storaged.pid", g_fdfs_base_path); + if ((result=process_action(pidFilename, argv[2], &stop)) != 0) + { + if (result == EINVAL) + { + usage(argv[0]); + } + log_destroy(); + return result; + } + if (stop) + { + log_destroy(); + return 0; + } + +#if defined(DEBUG_FLAG) && defined(OS_LINUX) + if (getExeAbsoluteFilename(argv[0], g_exe_name, \ + sizeof(g_exe_name)) == NULL) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return errno != 0 ? errno : ENOENT; + } +#endif + + memset(g_bind_addr, 0, sizeof(g_bind_addr)); + if ((result=storage_func_init(conf_filename, \ + g_bind_addr, sizeof(g_bind_addr))) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + sock = socketServer(g_bind_addr, g_server_port, &result); + if (sock < 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + if ((result=tcpsetserveropt(sock, g_fdfs_network_timeout)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + daemon_init(true); + umask(0); + if ((result=write_to_pid_file(pidFilename)) != 0) + { + log_destroy(); + return result; + } + + if (dup2(g_log_context.log_fd, STDOUT_FILENO) < 0 || \ + dup2(g_log_context.log_fd, STDERR_FILENO) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call dup2 fail, errno: %d, error info: %s, " \ + "program exit!", __LINE__, errno, STRERROR(errno)); + g_continue_flag = false; + return errno; + } + + if ((result=storage_sync_init()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_sync_init fail, program exit!", __LINE__); + g_continue_flag = false; + return result; + } + + if ((result=tracker_report_init()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "tracker_report_init fail, program exit!", __LINE__); + g_continue_flag = false; + return result; + } + + if ((result=storage_service_init()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_service_init fail, program exit!", __LINE__); + g_continue_flag = false; + return result; + } + + if ((result=set_rand_seed()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "set_rand_seed fail, program exit!", __LINE__); + g_continue_flag = false; + return result; + } + + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + + act.sa_handler = sigUsrHandler; + if(sigaction(SIGUSR1, &act, NULL) < 0 || \ + sigaction(SIGUSR2, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + + act.sa_handler = sigHupHandler; + if(sigaction(SIGHUP, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + + act.sa_handler = SIG_IGN; + if(sigaction(SIGPIPE, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + + act.sa_handler = sigQuitHandler; + if(sigaction(SIGINT, &act, NULL) < 0 || \ + sigaction(SIGTERM, &act, NULL) < 0 || \ + sigaction(SIGQUIT, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + +#if defined(DEBUG_FLAG) + +/* +#if defined(OS_LINUX) + memset(&act, 0, sizeof(act)); + act.sa_sigaction = sigSegvHandler; + act.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &act, NULL) < 0 || \ + sigaction(SIGABRT, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } +#endif +*/ + + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + act.sa_handler = sigDumpHandler; + if(sigaction(SIGUSR1, &act, NULL) < 0 || \ + sigaction(SIGUSR2, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } +#endif + +#ifdef WITH_HTTPD + if (!g_http_params.disabled) + { + if ((result=storage_httpd_start(g_bind_addr)) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_httpd_start fail, " \ + "program exit!", __LINE__); + return result; + } + } +#endif + + if ((result=tracker_report_thread_start()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "tracker_report_thread_start fail, " \ + "program exit!", __LINE__); + g_continue_flag = false; + storage_func_destroy(); + log_destroy(); + return result; + } + + scheduleArray.entries = scheduleEntries; + + memset(scheduleEntries, 0, sizeof(scheduleEntries)); + scheduleEntries[0].id = 1; + scheduleEntries[0].time_base.hour = TIME_NONE; + scheduleEntries[0].time_base.minute = TIME_NONE; + scheduleEntries[0].interval = g_sync_log_buff_interval; + scheduleEntries[0].task_func = log_sync_func; + scheduleEntries[0].func_args = &g_log_context; + + scheduleEntries[1].id = 2; + scheduleEntries[1].time_base.hour = TIME_NONE; + scheduleEntries[1].time_base.minute = TIME_NONE; + scheduleEntries[1].interval = g_sync_binlog_buff_interval; + scheduleEntries[1].task_func = fdfs_binlog_sync_func; + scheduleEntries[1].func_args = NULL; + + scheduleEntries[2].id = 3; + scheduleEntries[2].time_base.hour = TIME_NONE; + scheduleEntries[2].time_base.minute = TIME_NONE; + scheduleEntries[2].interval = g_sync_stat_file_interval; + scheduleEntries[2].task_func = fdfs_stat_file_sync_func; + scheduleEntries[2].func_args = NULL; + + scheduleArray.count = 3; + if (g_if_use_trunk_file) + { + scheduleEntries[scheduleArray.count].id = 4; + scheduleEntries[scheduleArray.count].time_base.hour = TIME_NONE; + scheduleEntries[scheduleArray.count].time_base.minute=TIME_NONE; + scheduleEntries[scheduleArray.count].interval = 1; + scheduleEntries[scheduleArray.count].task_func = \ + trunk_binlog_sync_func; + scheduleEntries[scheduleArray.count].func_args = NULL; + scheduleArray.count++; + } + + if (g_use_access_log) + { + scheduleEntries[scheduleArray.count].id = 5; + scheduleEntries[scheduleArray.count].time_base.hour = TIME_NONE; + scheduleEntries[scheduleArray.count].time_base.minute=TIME_NONE; + scheduleEntries[scheduleArray.count].interval = \ + g_sync_log_buff_interval; + scheduleEntries[scheduleArray.count].task_func = log_sync_func; + scheduleEntries[scheduleArray.count].func_args = \ + &g_access_log_context; + scheduleArray.count++; + + if (g_rotate_access_log) + { + scheduleEntries[scheduleArray.count].id = 6; + scheduleEntries[scheduleArray.count].time_base = \ + g_access_log_rotate_time; + scheduleEntries[scheduleArray.count].interval = \ + 24 * 3600; + scheduleEntries[scheduleArray.count].task_func = \ + log_notify_rotate; + scheduleEntries[scheduleArray.count].func_args = \ + &g_access_log_context; + scheduleArray.count++; + } + } + + if (g_rotate_error_log) + { + scheduleEntries[scheduleArray.count].id = 7; + scheduleEntries[scheduleArray.count].time_base = \ + g_error_log_rotate_time; + scheduleEntries[scheduleArray.count].interval = \ + 24 * 3600; + scheduleEntries[scheduleArray.count].task_func = \ + log_notify_rotate; + scheduleEntries[scheduleArray.count].func_args = \ + &g_log_context; + scheduleArray.count++; + } + + if ((result=sched_start(&scheduleArray, &schedule_tid, \ + g_thread_stack_size, (bool * volatile)&g_continue_flag)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + if ((result=set_run_by(g_run_by_group, g_run_by_user)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + if ((result=storage_dio_init()) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + log_set_cache(true); + + bTerminateFlag = false; + bAcceptEndFlag = false; + + storage_accept_loop(sock); + bAcceptEndFlag = true; + + fdfs_binlog_sync_func(NULL); //binlog fsync + + if (g_schedule_flag) + { + pthread_kill(schedule_tid, SIGINT); + } + + storage_terminate_threads(); + storage_dio_terminate(); + + kill_tracker_report_threads(); + kill_storage_sync_threads(); + + wait_count = 0; + while (g_storage_thread_count != 0 || \ + g_dio_thread_count != 0 || \ + g_tracker_reporter_count > 0 || \ + g_schedule_flag) + { +/* +#if defined(DEBUG_FLAG) && defined(OS_LINUX) + if (bSegmentFault) + { + sleep(5); + break; + } +#endif +*/ + + usleep(10000); + if (++wait_count > 6000) + { + logWarning("waiting timeout, exit!"); + break; + } + } + + tracker_report_destroy(); + storage_service_destroy(); + storage_sync_destroy(); + storage_func_destroy(); + + if (g_if_use_trunk_file) + { + trunk_sync_destroy(); + storage_trunk_destroy(); + } + + logInfo("exit normally.\n"); + log_destroy(); + + delete_pid_file(pidFilename); + return 0; +} + +static void sigQuitHandler(int sig) +{ + if (!bTerminateFlag) + { + set_timer(1, 1, sigAlarmHandler); + + bTerminateFlag = true; + g_continue_flag = false; + + logCrit("file: "__FILE__", line: %d, " \ + "catch signal %d, program exiting...", \ + __LINE__, sig); + } +} + +static void sigAlarmHandler(int sig) +{ + ConnectionInfo server; + + if (bAcceptEndFlag) + { + return; + } + + logDebug("file: "__FILE__", line: %d, " \ + "signal server to quit...", __LINE__); + + if (*g_bind_addr != '\0') + { + strcpy(server.ip_addr, g_bind_addr); + } + else + { + strcpy(server.ip_addr, "127.0.0.1"); + } + server.port = g_server_port; + server.sock = -1; + + if (conn_pool_connect_server(&server, g_fdfs_connect_timeout) != 0) + { + return; + } + + fdfs_quit(&server); + conn_pool_disconnect_server(&server); + + logDebug("file: "__FILE__", line: %d, " \ + "signal server to quit done", __LINE__); +} + +static void sigHupHandler(int sig) +{ + if (g_rotate_error_log) + { + g_log_context.rotate_immediately = true; + } + + if (g_rotate_access_log) + { + g_access_log_context.rotate_immediately = true; + } + + logInfo("file: "__FILE__", line: %d, " \ + "catch signal %d, rotate log", __LINE__, sig); +} + +static void sigUsrHandler(int sig) +{ + logInfo("file: "__FILE__", line: %d, " \ + "catch signal %d, ignore it", __LINE__, sig); +} + +#if defined(DEBUG_FLAG) + +/* +#if defined(OS_LINUX) +static void sigSegvHandler(int signum, siginfo_t *info, void *ptr) +{ + bSegmentFault = true; + + if (!bTerminateFlag) + { + set_timer(1, 1, sigAlarmHandler); + + bTerminateFlag = true; + g_continue_flag = false; + + logCrit("file: "__FILE__", line: %d, " \ + "catch signal %d, program exiting...", \ + __LINE__, signum); + + signal_stack_trace_print(signum, info, ptr); + } +} +#endif +*/ + +static void sigDumpHandler(int sig) +{ + static bool bDumpFlag = false; + char filename[MAX_PATH_SIZE]; + + if (bDumpFlag) + { + return; + } + + bDumpFlag = true; + + snprintf(filename, sizeof(filename), + "%s/logs/storage_dump.log", g_fdfs_base_path); + fdfs_dump_storage_global_vars_to_file(filename); + + bDumpFlag = false; +} +#endif + diff --git a/storage/fdht_client/fdht_client.c b/storage/fdht_client/fdht_client.c new file mode 100644 index 0000000..b3769f2 --- /dev/null +++ b/storage/fdht_client/fdht_client.c @@ -0,0 +1,1473 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include "sockopt.h" +#include "logger.h" +#include "hash.h" +#include "shared_func.h" +#include "ini_file_reader.h" +#include "fdht_types.h" +#include "fdht_proto.h" +#include "fdht_global.h" +#include "fdht_client.h" + +GroupArray g_group_array = {NULL, 0}; +bool g_keep_alive = false; + +static void fdht_proxy_extra_deal(GroupArray *pGroupArray, bool *bKeepAlive) +{ + int group_id; + ServerArray *pServerArray; + FDHTServerInfo **ppServer; + FDHTServerInfo **ppServerEnd; + + if (!pGroupArray->use_proxy) + { + return; + } + + *bKeepAlive = true; + pGroupArray->server_count = 1; + memcpy(pGroupArray->servers, &pGroupArray->proxy_server, \ + sizeof(FDHTServerInfo)); + + pServerArray = pGroupArray->groups; + for (group_id=0; group_idgroup_count; group_id++) + { + ppServerEnd = pServerArray->servers + pServerArray->count; + for (ppServer=pServerArray->servers; \ + ppServerservers; + } + + pServerArray++; + } +} + +int fdht_client_init(const char *filename) +{ + char *pBasePath; + IniContext iniContext; + char szProxyPrompt[64]; + int result; + + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(filename, &iniContext)) != 0) + { + logError("load conf file \"%s\" fail, ret code: %d", \ + filename, result); + return result; + } + + //iniPrintItems(&iniContext); + + while (1) + { + pBasePath = iniGetStrValue(NULL, "base_path", &iniContext); + if (pBasePath == NULL) + { + logError("conf file \"%s\" must have item " \ + "\"base_path\"!", filename); + result = ENOENT; + break; + } + + snprintf(g_fdht_base_path, sizeof(g_fdht_base_path), "%s", pBasePath); + chopPath(g_fdht_base_path); + if (!fileExists(g_fdht_base_path)) + { + logError("\"%s\" can't be accessed, error info: %s", \ + g_fdht_base_path, STRERROR(errno)); + result = errno != 0 ? errno : ENOENT; + break; + } + if (!isDir(g_fdht_base_path)) + { + logError("\"%s\" is not a directory!", g_fdht_base_path); + result = ENOTDIR; + break; + } + + g_fdht_connect_timeout = iniGetIntValue(NULL, "connect_timeout", \ + &iniContext, DEFAULT_CONNECT_TIMEOUT); + if (g_fdht_connect_timeout <= 0) + { + g_fdht_connect_timeout = DEFAULT_CONNECT_TIMEOUT; + } + + g_fdht_network_timeout = iniGetIntValue(NULL, "network_timeout", \ + &iniContext, DEFAULT_NETWORK_TIMEOUT); + if (g_fdht_network_timeout <= 0) + { + g_fdht_network_timeout = DEFAULT_NETWORK_TIMEOUT; + } + + g_keep_alive = iniGetBoolValue(NULL, "keep_alive", \ + &iniContext, false); + + if ((result=fdht_load_groups(&iniContext, \ + &g_group_array)) != 0) + { + break; + } + + if (g_group_array.use_proxy) + { + sprintf(szProxyPrompt, "proxy_addr=%s, proxy_port=%d, ", + g_group_array.proxy_server.ip_addr, + g_group_array.proxy_server.port); + } + else + { + *szProxyPrompt = '\0'; + } + + load_log_level(&iniContext); + + logInfo("file: "__FILE__", line: %d, " \ + "base_path=%s, " \ + "connect_timeout=%ds, network_timeout=%ds, " \ + "keep_alive=%d, use_proxy=%d, %s"\ + "group_count=%d, server_count=%d", __LINE__, \ + g_fdht_base_path, g_fdht_connect_timeout, \ + g_fdht_network_timeout, g_keep_alive, \ + g_group_array.use_proxy, szProxyPrompt, \ + g_group_array.group_count, g_group_array.server_count); + + fdht_proxy_extra_deal(&g_group_array, &g_keep_alive); + + break; + } + + iniFreeContext(&iniContext); + + return result; +} + +int fdht_load_conf(const char *filename, GroupArray *pGroupArray, \ + bool *bKeepAlive) +{ + IniContext iniContext; + int result; + + if ((result=iniLoadFromFile(filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, " \ + "ret code: %d", __LINE__, \ + filename, result); + return result; + } + + *bKeepAlive = iniGetBoolValue(NULL, "keep_alive", &iniContext, false); + if ((result=fdht_load_groups(&iniContext, pGroupArray)) != 0) + { + iniFreeContext(&iniContext); + return result; + } + + fdht_proxy_extra_deal(pGroupArray, bKeepAlive); + + iniFreeContext(&iniContext); + return 0; +} + +void fdht_client_destroy() +{ + fdht_free_group_array(&g_group_array); +} + +#define get_readable_connection(pServerArray, bKeepAlive, hash_code, err_no) \ + get_connection(pServerArray, bKeepAlive, hash_code, err_no) + +#define get_writable_connection(pServerArray, bKeepAlive, hash_code, err_no) \ + get_connection(pServerArray, bKeepAlive, hash_code, err_no) + +static FDHTServerInfo *get_connection(ServerArray *pServerArray, \ + const bool bKeepAlive, const int hash_code, int *err_no) +{ + FDHTServerInfo **ppServer; + FDHTServerInfo **ppEnd; + int server_index; + int new_hash_code; + + new_hash_code = (hash_code << 16) | (hash_code >> 16); + if (new_hash_code < 0) + { + new_hash_code &= 0x7FFFFFFF; + } + server_index = new_hash_code % pServerArray->count; + ppEnd = pServerArray->servers + pServerArray->count; + for (ppServer = pServerArray->servers + server_index; \ + ppServersock > 0) //already connected + { + return *ppServer; + } + + if (fdht_connect_server_nb(*ppServer, \ + g_fdht_connect_timeout) == 0) + { + if (bKeepAlive) + { + tcpsetnodelay((*ppServer)->sock, 3600); + } + return *ppServer; + } + } + + ppEnd = pServerArray->servers + server_index; + for (ppServer = pServerArray->servers; ppServersock > 0) //already connected + { + return *ppServer; + } + + if (fdht_connect_server_nb(*ppServer, \ + g_fdht_connect_timeout) == 0) + { + if (bKeepAlive) + { + tcpsetnodelay((*ppServer)->sock, 3600); + } + return *ppServer; + } + } + + *err_no = ENOENT; + return NULL; +} + +#define CALC_KEY_HASH_CODE(pKeyInfo, hash_key, hash_key_len, key_hash_code) \ + if (pKeyInfo->namespace_len > FDHT_MAX_NAMESPACE_LEN) \ + { \ + fprintf(stderr, "namespace length: %d exceeds, " \ + "max length: %d\n", \ + pKeyInfo->namespace_len, FDHT_MAX_NAMESPACE_LEN); \ + return EINVAL; \ + } \ + \ + if (pKeyInfo->obj_id_len > FDHT_MAX_OBJECT_ID_LEN) \ + { \ + fprintf(stderr, "object ID length: %d exceeds, " \ + "max length: %d\n", \ + pKeyInfo->obj_id_len, FDHT_MAX_OBJECT_ID_LEN); \ + return EINVAL; \ + } \ + \ + if (pKeyInfo->key_len > FDHT_MAX_SUB_KEY_LEN) \ + { \ + fprintf(stderr, "key length: %d exceeds, max length: %d\n", \ + pKeyInfo->key_len, FDHT_MAX_SUB_KEY_LEN); \ + return EINVAL; \ + } \ + \ + if (pKeyInfo->namespace_len == 0 && pKeyInfo->obj_id_len == 0) \ + { \ + hash_key_len = pKeyInfo->key_len; \ + memcpy(hash_key, pKeyInfo->szKey, pKeyInfo->key_len); \ + } \ + else if (pKeyInfo->namespace_len > 0 && pKeyInfo->obj_id_len > 0) \ + { \ + hash_key_len = pKeyInfo->namespace_len+1+pKeyInfo->obj_id_len; \ + memcpy(hash_key,pKeyInfo->szNameSpace,pKeyInfo->namespace_len);\ + *(hash_key + pKeyInfo->namespace_len)=FDHT_FULL_KEY_SEPERATOR; \ + memcpy(hash_key + pKeyInfo->namespace_len + 1, \ + pKeyInfo->szObjectId, pKeyInfo->obj_id_len); \ + } \ + else \ + { \ + fprintf(stderr, "invalid namespace length: %d and " \ + "object ID length: %d\n", \ + pKeyInfo->namespace_len, pKeyInfo->obj_id_len); \ + return EINVAL; \ + } \ + \ + key_hash_code = PJWHash(hash_key, hash_key_len); \ + if (key_hash_code < 0) \ + { \ + key_hash_code &= 0x7FFFFFFF; \ + } \ + + +#define CALC_OBJECT_HASH_CODE(pObjectInfo, hash_key, hash_key_len, key_hash_code) \ + if (pObjectInfo->namespace_len <= 0 || pObjectInfo->obj_id_len <= 0) \ + { \ + fprintf(stderr, "invalid namespace length: %d and " \ + "object ID length: %d\n", \ + pObjectInfo->namespace_len, pObjectInfo->obj_id_len); \ + return EINVAL; \ + } \ + \ + if (pObjectInfo->namespace_len > FDHT_MAX_NAMESPACE_LEN) \ + { \ + fprintf(stderr, "namespace length: %d exceeds, " \ + "max length: %d\n", \ + pObjectInfo->namespace_len, FDHT_MAX_NAMESPACE_LEN); \ + return EINVAL; \ + } \ + \ + if (pObjectInfo->obj_id_len > FDHT_MAX_OBJECT_ID_LEN) \ + { \ + fprintf(stderr, "object ID length: %d exceeds, " \ + "max length: %d\n", \ + pObjectInfo->obj_id_len, FDHT_MAX_OBJECT_ID_LEN); \ + return EINVAL; \ + } \ + hash_key_len = pObjectInfo->namespace_len+1+pObjectInfo->obj_id_len; \ + memcpy(hash_key, pObjectInfo->szNameSpace, pObjectInfo->namespace_len);\ + *(hash_key + pObjectInfo->namespace_len) = FDHT_FULL_KEY_SEPERATOR; \ + memcpy(hash_key + pObjectInfo->namespace_len + 1, \ + pObjectInfo->szObjectId, pObjectInfo->obj_id_len); \ + \ + key_hash_code = PJWHash(hash_key, hash_key_len); \ + if (key_hash_code < 0) \ + { \ + key_hash_code &= 0x7FFFFFFF; \ + } \ + + +/** +* request body format: +* namespace_len: 4 bytes big endian integer +* namespace: can be emtpy +* obj_id_len: 4 bytes big endian integer +* object_id: the object id (can be empty) +* key_len: 4 bytes big endian integer +* key: key name +* response body format: +* value_len: 4 bytes big endian integer +* value: value buff +*/ +int fdht_get_ex1(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo, const time_t expires, \ + char **ppValue, int *value_len, MallocFunc malloc_func) +{ + int result; + FDHTProtoHeader *pHeader; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + char buff[sizeof(FDHTProtoHeader) + FDHT_MAX_FULL_KEY_LEN + 16]; + int in_bytes; + int vlen; + int group_id; + int hash_key_len; + int key_hash_code; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + char *p; + + CALC_KEY_HASH_CODE(pKeyInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + //printf("get group_id=%d\n", group_id); + + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_readable_connection(pGroup, bKeepAlive, \ + key_hash_code, &result); + if (pServer == NULL) + { + return result; + } + + memset(buff, 0, sizeof(buff)); + pHeader = (FDHTProtoHeader *)buff; + + pHeader->cmd = FDHT_PROTO_CMD_GET; + pHeader->keep_alive = bKeepAlive; + int2buff((int)time(NULL), pHeader->timestamp); + int2buff((int)expires, pHeader->expires); + int2buff(key_hash_code, pHeader->key_hash_code); + int2buff(12 + pKeyInfo->namespace_len + pKeyInfo->obj_id_len + \ + pKeyInfo->key_len, pHeader->pkg_len); + + do + { + p = buff + sizeof(FDHTProtoHeader); + PACK_BODY_UNTIL_KEY(pKeyInfo, p) + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("send data to server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes < 4) + { + logError("server %s:%d reponse bytes: %d < 4", \ + pServer->ip_addr, pServer->port, in_bytes); + result = EINVAL; + break; + } + + if ((result=tcprecvdata_nb(pServer->sock, buff, \ + 4, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, \ + pServer->port, \ + result, STRERROR(result)); + break; + } + + vlen = buff2int(buff); + if (vlen != in_bytes - 4) + { + logError("server %s:%d reponse bytes: %d " \ + "is not correct, %d != %d", pServer->ip_addr, \ + pServer->port, in_bytes, vlen, in_bytes - 4); + result = EINVAL; + break; + } + + if (*ppValue != NULL) + { + if (vlen >= *value_len) + { + *value_len = 0; + result = ENOSPC; + break; + } + + *value_len = vlen; + } + else + { + *value_len = vlen; + *ppValue = (char *)malloc_func((*value_len + 1)); + if (*ppValue == NULL) + { + *value_len = 0; + logError("malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + *value_len + 1, errno, STRERROR(errno)); + result = errno != 0 ? errno : ENOMEM; + break; + } + } + + if ((result=tcprecvdata_nb(pServer->sock, *ppValue, \ + *value_len, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, \ + pServer->port, \ + result, STRERROR(result)); + break; + } + + *(*ppValue + *value_len) = '\0'; + } while(0); + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + +int fdht_batch_set_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTObjectInfo *pObjectInfo, FDHTKeyValuePair *key_list, \ + const int key_count, const time_t expires, int *success_count) +{ + int result; + FDHTProtoHeader *pHeader; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + char buff[sizeof(FDHTProtoHeader) + FDHT_MAX_FULL_KEY_LEN + \ + (8 + FDHT_MAX_SUB_KEY_LEN) * FDHT_MAX_KEY_COUNT_PER_REQ + \ + 32 * 1024]; + char *pBuff; + int in_bytes; + int total_key_len; + int total_value_len; + int pkg_total_len; + int group_id; + int hash_key_len; + int key_hash_code; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + FDHTKeyValuePair *pKeyValuePair; + FDHTKeyValuePair *pKeyValueEnd; + char *p; + + *success_count = 0; + if (key_count <= 0 || key_count > FDHT_MAX_KEY_COUNT_PER_REQ) + { + logError("invalid key_count: %d", key_count); + return EINVAL; + } + + CALC_OBJECT_HASH_CODE(pObjectInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_writable_connection(pGroup, bKeepAlive, \ + key_hash_code, &result); + if (pServer == NULL) + { + return result; + } + + total_key_len = 0; + total_value_len = 0; + pKeyValueEnd = key_list + key_count; + for (pKeyValuePair=key_list; pKeyValuePairkey_len; + total_value_len += pKeyValuePair->value_len; + } + pkg_total_len = sizeof(FDHTProtoHeader) + 12 + pObjectInfo->namespace_len + \ + pObjectInfo->obj_id_len + 8 * key_count + \ + total_key_len + total_value_len; + + if (pkg_total_len <= sizeof(buff)) + { + pBuff = buff; + } + else + { + pBuff = (char *)malloc(pkg_total_len); + if (pBuff == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + pkg_total_len, result, STRERROR(result)); + return result; + } + } + + memset(pBuff, 0, pkg_total_len); + pHeader = (FDHTProtoHeader *)pBuff; + + pHeader->cmd = FDHT_PROTO_CMD_BATCH_SET; + pHeader->keep_alive = bKeepAlive; + int2buff((int)time(NULL), pHeader->timestamp); + int2buff((int)expires, pHeader->expires); + int2buff(key_hash_code, pHeader->key_hash_code); + + p = pBuff + sizeof(FDHTProtoHeader); + PACK_BODY_OBJECT(pObjectInfo, p) + int2buff(key_count, p); + p += 4; + + for (pKeyValuePair=key_list; pKeyValuePairkey_len, p); + memcpy(p + 4, pKeyValuePair->szKey, pKeyValuePair->key_len); + p += 4 + pKeyValuePair->key_len; + + int2buff(pKeyValuePair->value_len, p); + memcpy(p + 4, pKeyValuePair->pValue, pKeyValuePair->value_len); + p += 4 + pKeyValuePair->value_len; + } + + do + { + int2buff(pkg_total_len - sizeof(FDHTProtoHeader), pHeader->pkg_len); + if ((result=tcpsenddata_nb(pServer->sock, pBuff, pkg_total_len, \ + g_fdht_network_timeout)) != 0) + { + logError("send data to server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes != 8 + 5 * key_count + total_key_len) + { + logError("server %s:%d reponse bytes: %d != %d", \ + pServer->ip_addr, pServer->port, in_bytes, \ + 8 + 5 * key_count + total_key_len); + result = EINVAL; + break; + } + + if ((result=tcprecvdata_nb(pServer->sock, pBuff, \ + in_bytes, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if (buff2int(pBuff) != key_count) + { + result = EINVAL; + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, invalid key_count: %d, " \ + "expect key count: %d", \ + __LINE__, pServer->ip_addr, pServer->port, \ + buff2int(pBuff), key_count); + break; + } + + *success_count = buff2int(pBuff + 4); + p = pBuff + 8; + for (pKeyValuePair=key_list; pKeyValuePairkey_len = buff2int(p); + + memcpy(pKeyValuePair->szKey, p + 4, \ + pKeyValuePair->key_len); + p += 4 + pKeyValuePair->key_len; + pKeyValuePair->status = *p++; + } + } while (0); + + if (pBuff != buff) + { + free(pBuff); + } + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + break; + } + + return result; +} + +int fdht_batch_delete_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTObjectInfo *pObjectInfo, FDHTKeyValuePair *key_list, \ + const int key_count, int *success_count) +{ + int result; + FDHTProtoHeader *pHeader; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + char buff[sizeof(FDHTProtoHeader) + FDHT_MAX_FULL_KEY_LEN + 8 + \ + (5 + FDHT_MAX_SUB_KEY_LEN) * FDHT_MAX_KEY_COUNT_PER_REQ]; + int in_bytes; + int total_key_len; + int group_id; + int hash_key_len; + int key_hash_code; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + FDHTKeyValuePair *pKeyValuePair; + FDHTKeyValuePair *pKeyValueEnd; + char *p; + + *success_count = 0; + if (key_count <= 0 || key_count > FDHT_MAX_KEY_COUNT_PER_REQ) + { + logError("invalid key_count: %d", key_count); + return EINVAL; + } + + CALC_OBJECT_HASH_CODE(pObjectInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_readable_connection(pGroup, bKeepAlive, \ + key_hash_code, &result); + if (pServer == NULL) + { + return result; + } + + memset(buff, 0, sizeof(buff)); + pHeader = (FDHTProtoHeader *)buff; + + pHeader->cmd = FDHT_PROTO_CMD_BATCH_DEL; + pHeader->keep_alive = bKeepAlive; + int2buff((int)time(NULL), pHeader->timestamp); + int2buff(key_hash_code, pHeader->key_hash_code); + + p = buff + sizeof(FDHTProtoHeader); + PACK_BODY_OBJECT(pObjectInfo, p) + int2buff(key_count, p); + p += 4; + + total_key_len = 0; + pKeyValueEnd = key_list + key_count; + for (pKeyValuePair=key_list; pKeyValuePairkey_len, p); + memcpy(p + 4, pKeyValuePair->szKey, pKeyValuePair->key_len); + p += 4 + pKeyValuePair->key_len; + + total_key_len += pKeyValuePair->key_len; + } + + do + { + int2buff((p - buff) - sizeof(FDHTProtoHeader), pHeader->pkg_len); + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("send data to server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes != 8 + 5 * key_count + total_key_len) + { + logError("server %s:%d reponse bytes: %d != %d", \ + pServer->ip_addr, pServer->port, in_bytes, \ + 8 + 5 * key_count + total_key_len); + result = EINVAL; + break; + } + + if ((result=tcprecvdata_nb(pServer->sock, buff, \ + in_bytes, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if (buff2int(buff) != key_count) + { + result = EINVAL; + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, invalid key_count: %d, " \ + "expect key count: %d", \ + __LINE__, pServer->ip_addr, pServer->port, \ + buff2int(buff), key_count); + break; + } + + *success_count = buff2int(buff + 4); + p = buff + 8; + for (pKeyValuePair=key_list; pKeyValuePairkey_len = buff2int(p); + + memcpy(pKeyValuePair->szKey, p + 4, \ + pKeyValuePair->key_len); + p += 4 + pKeyValuePair->key_len; + pKeyValuePair->status = *p++; + } + } while (0); + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + +int fdht_batch_get_ex1(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTObjectInfo *pObjectInfo, FDHTKeyValuePair *key_list, \ + const int key_count, const time_t expires, \ + MallocFunc malloc_func, int *success_count) +{ + int result; + FDHTProtoHeader *pHeader; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + char buff[sizeof(FDHTProtoHeader) + FDHT_MAX_FULL_KEY_LEN + \ + (4 + FDHT_MAX_SUB_KEY_LEN) * FDHT_MAX_KEY_COUNT_PER_REQ + \ + 32 * 1024]; + int in_bytes; + int value_len; + int group_id; + int hash_key_len; + int key_hash_code; + char *pInBuff; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + FDHTKeyValuePair *pKeyValuePair; + FDHTKeyValuePair *pKeyValueEnd; + char *p; + + *success_count = 0; + if (key_count <= 0 || key_count > FDHT_MAX_KEY_COUNT_PER_REQ) + { + logError("invalid key_count: %d", key_count); + return EINVAL; + } + + CALC_OBJECT_HASH_CODE(pObjectInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_readable_connection(pGroup, bKeepAlive, \ + key_hash_code, &result); + if (pServer == NULL) + { + return result; + } + + memset(buff, 0, sizeof(buff)); + pHeader = (FDHTProtoHeader *)buff; + + pHeader->cmd = FDHT_PROTO_CMD_BATCH_GET; + pHeader->keep_alive = bKeepAlive; + int2buff((int)time(NULL), pHeader->timestamp); + int2buff((int)expires, pHeader->expires); + int2buff(key_hash_code, pHeader->key_hash_code); + + p = buff + sizeof(FDHTProtoHeader); + PACK_BODY_OBJECT(pObjectInfo, p) + int2buff(key_count, p); + p += 4; + + pKeyValueEnd = key_list + key_count; + for (pKeyValuePair=key_list; pKeyValuePairkey_len, p); + memcpy(p + 4, pKeyValuePair->szKey, pKeyValuePair->key_len); + p += 4 + pKeyValuePair->key_len; + } + + pInBuff = buff; + do + { + int2buff((p - buff) - sizeof(FDHTProtoHeader), pHeader->pkg_len); + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("send data to server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes < 17) + { + logError("server %s:%d reponse bytes: %d < 17", \ + pServer->ip_addr, pServer->port, in_bytes); + result = EINVAL; + break; + } + + if (in_bytes > sizeof(buff)) + { + pInBuff = (char *)malloc(in_bytes); + if (pInBuff == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, in_bytes, \ + result, STRERROR(result)); + break; + } + } + + if ((result=tcprecvdata_nb(pServer->sock, pInBuff, \ + in_bytes, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if (buff2int(pInBuff) != key_count) + { + result = EINVAL; + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, invalid key_count: %d, " \ + "expect key count: %d", \ + __LINE__, pServer->ip_addr, pServer->port, \ + buff2int(pInBuff), key_count); + break; + } + + *success_count = buff2int(pInBuff + 4); + p = pInBuff + 8; + for (pKeyValuePair=key_list; pKeyValuePairkey_len = buff2int(p); + + memcpy(pKeyValuePair->szKey, p + 4, \ + pKeyValuePair->key_len); + p += 4 + pKeyValuePair->key_len; + pKeyValuePair->status = *p++; + if (pKeyValuePair->status != 0) + { + pKeyValuePair->value_len = 0; + continue; + } + + value_len = buff2int(p); + p += 4; + if (pKeyValuePair->pValue != NULL) + { + if (value_len >= pKeyValuePair->value_len) + { + *(pKeyValuePair->pValue) = '\0'; + pKeyValuePair->value_len = 0; + pKeyValuePair->status = ENOSPC; + } + else + { + pKeyValuePair->value_len = value_len; + memcpy(pKeyValuePair->pValue, p, \ + value_len); + *(pKeyValuePair->pValue+value_len)='\0'; + } + } + else + { + pKeyValuePair->pValue = (char *)malloc_func( \ + value_len + 1); + if (pKeyValuePair->pValue == NULL) + { + pKeyValuePair->value_len = 0; + pKeyValuePair->status = errno != 0 ? \ + errno : ENOMEM; + logError("malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + value_len+1, errno, \ + STRERROR(errno)); + } + else + { + pKeyValuePair->value_len = value_len; + memcpy(pKeyValuePair->pValue, p, \ + value_len); + *(pKeyValuePair->pValue+value_len)='\0'; + } + } + + p += value_len; + } + + if (in_bytes != p - pInBuff) + { + *success_count = 0; + logError("server %s:%d reponse bytes: %d != %d", \ + pServer->ip_addr, pServer->port, \ + in_bytes, (int)(p - pInBuff)); + result = EINVAL; + break; + } + } while (0); + + if (pInBuff != buff) + { + free(pInBuff); + } + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + +int fdht_set_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo, const time_t expires, \ + const char *pValue, const int value_len) +{ + int result; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + int group_id; + int hash_key_len; + int key_hash_code; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + + CALC_KEY_HASH_CODE(pKeyInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_writable_connection(pGroup, bKeepAlive, \ + key_hash_code, &result); + if (pServer == NULL) + { + return result; + } + + //printf("key_hash_code=%d, group_id=%d\n", key_hash_code, group_id); + + //printf("set group_id=%d\n", group_id); + result = fdht_client_set(pServer, bKeepAlive, time(NULL), expires, \ + FDHT_PROTO_CMD_SET, key_hash_code, \ + pKeyInfo, pValue, value_len); + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + +/** +* request body format: +* namespace_len: 4 bytes big endian integer +* namespace: can be emtpy +* obj_id_len: 4 bytes big endian integer +* object_id: the object id (can be empty) +* key_len: 4 bytes big endian integer +* key: key name +* incr 4 bytes big endian integer +* response body format: +* value_len: 4 bytes big endian integer +* value : value_len bytes +*/ +int fdht_inc_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo, const time_t expires, \ + const int increase, char *pValue, int *value_len) +{ + int result; + FDHTProtoHeader *pHeader; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + char buff[FDHT_MAX_FULL_KEY_LEN + 32]; + char *in_buff; + int in_bytes; + int group_id; + int hash_key_len; + int key_hash_code; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + char *p; + + CALC_KEY_HASH_CODE(pKeyInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_writable_connection(pGroup, bKeepAlive, \ + key_hash_code, &result); + if (pServer == NULL) + { + return result; + } + + //printf("inc group_id=%d\n", group_id); + + memset(buff, 0, sizeof(buff)); + pHeader = (FDHTProtoHeader *)buff; + + pHeader->cmd = FDHT_PROTO_CMD_INC; + pHeader->keep_alive = bKeepAlive; + int2buff((int)time(NULL), pHeader->timestamp); + int2buff((int)expires, pHeader->expires); + int2buff(key_hash_code, pHeader->key_hash_code); + int2buff(16 + pKeyInfo->namespace_len + pKeyInfo->obj_id_len + \ + pKeyInfo->key_len, pHeader->pkg_len); + + while (1) + { + p = buff + sizeof(FDHTProtoHeader); + PACK_BODY_UNTIL_KEY(pKeyInfo, p) + int2buff(increase, p); + p += 4; + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("send data to server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + in_buff = buff; + if ((result=fdht_recv_response(pServer, &in_buff, \ + sizeof(buff), &in_bytes)) != 0) + { + logError("recv data from server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if (in_bytes < 4) + { + logError("server %s:%d reponse bytes: %d < 4!", \ + pServer->ip_addr, pServer->port, in_bytes); + result = EINVAL; + break; + } + + if (in_bytes - 4 >= *value_len) + { + *value_len = 0; + result = ENOSPC; + break; + } + + *value_len = in_bytes - 4; + memcpy(pValue, in_buff + 4, *value_len); + *(pValue + (*value_len)) = '\0'; + break; + } + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + +int fdht_delete_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo) +{ + int result; + char hash_key[FDHT_MAX_FULL_KEY_LEN + 1]; + int group_id; + int hash_key_len; + int key_hash_code; + int i; + ServerArray *pGroup; + FDHTServerInfo *pServer; + + CALC_KEY_HASH_CODE(pKeyInfo, hash_key, hash_key_len, key_hash_code) + group_id = ((unsigned int)key_hash_code) % pGroupArray->group_count; + pGroup = pGroupArray->groups + group_id; + for (i=0; i<=pGroup->count; i++) + { + pServer = get_writable_connection(pGroup, bKeepAlive, \ + key_hash_code , &result); + if (pServer == NULL) + { + return result; + } + + //printf("del group_id=%d\n", group_id); + result = fdht_client_delete(pServer, bKeepAlive, time(NULL), \ + FDHT_PROTO_CMD_DEL, key_hash_code, pKeyInfo); + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + +int fdht_connect_all_servers(GroupArray *pGroupArray, const bool bKeepAlive, \ + int *success_count, int *fail_count) +{ + FDHTServerInfo *pServerInfo; + FDHTServerInfo *pServerEnd; + int conn_result; + int result; + + *success_count = 0; + *fail_count = 0; + if (pGroupArray->servers == NULL) + { + return ENOENT; + } + + result = 0; + + pServerEnd = pGroupArray->servers + pGroupArray->server_count; + for (pServerInfo=pGroupArray->servers; \ + pServerInfouse_proxy) + { + tcpsetnodelay(pServerInfo->sock, 3600); + } + } + } + + if (result != 0) + { + return result; + } + else + { + return *success_count > 0 ? 0: ENOENT; + } +} + +void fdht_disconnect_all_servers(GroupArray *pGroupArray) +{ + FDHTServerInfo *pServerInfo; + FDHTServerInfo *pServerEnd; + + if (pGroupArray->servers != NULL) + { + pServerEnd = pGroupArray->servers + pGroupArray->server_count; + for (pServerInfo=pGroupArray->servers; \ + pServerInfosock >= 0) + { + if (!pGroupArray->use_proxy) + { + fdht_quit(pServerInfo); + } + close(pServerInfo->sock); + pServerInfo->sock = -1; + } + } + } +} + +int fdht_stat_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + const int server_index, char *buff, const int size) +{ + int result; + int in_bytes; + int i; + FDHTProtoHeader header; + FDHTServerInfo *pServer; + + memset(buff, 0, size); + if (server_index < 0 || server_index > pGroupArray->server_count) + { + logError("invalid servier_index: %d", server_index); + return EINVAL; + } + + pServer = pGroupArray->servers + server_index; + for (i=0; i<2; i++) + { + if ((result=fdht_connect_server_nb(pServer, \ + g_fdht_connect_timeout)) != 0) + { + return result; + } + + if (bKeepAlive) + { + tcpsetnodelay(pServer->sock, 3600); + } + + memset(&header, 0, sizeof(header)); + header.cmd = FDHT_PROTO_CMD_STAT; + header.keep_alive = bKeepAlive; + int2buff((int)time(NULL), header.timestamp); + + do + { + if ((result=tcpsenddata_nb(pServer->sock, &header, \ + sizeof(header), g_fdht_network_timeout)) != 0) + { + logError("send data to server %s:%d fail, " \ + "errno: %d, error info: %s", \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + break; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + break; + } + + if (in_bytes >= size) + { + logError("server %s:%d reponse bytes: %d >= " \ + "buff size: %d", pServer->ip_addr, \ + pServer->port, in_bytes, size); + result = ENOSPC; + break; + } + + if ((result=tcprecvdata_nb(pServer->sock, buff, \ + in_bytes, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, \ + pServer->port, \ + result, STRERROR(result)); + break; + } + } while (0); + + if (bKeepAlive) + { + if (result >= ENETDOWN) //network error + { + fdht_disconnect_server(pServer); + if (result == ENOTCONN) + { + continue; //retry + } + } + } + else + { + fdht_disconnect_server(pServer); + } + + break; + } + + return result; +} + diff --git a/storage/fdht_client/fdht_client.h b/storage/fdht_client/fdht_client.h new file mode 100644 index 0000000..07222a8 --- /dev/null +++ b/storage/fdht_client/fdht_client.h @@ -0,0 +1,232 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_client.h + +#ifndef _FDHT_CLIENT_H +#define _FDHT_CLIENT_H + +#include +#include +#include +#include "fdht_define.h" +#include "fdht_types.h" +#include "fdht_proto.h" +#include "fdht_func.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern GroupArray g_group_array; //group info, including server list +extern bool g_keep_alive; //persistent connection flag + +/* +init function +param: + filename: client config filename +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_client_init(const char *filename); + + +/* +init function +param: + filename: client config filename + pGroupArray: return server list + bKeepAlive: return keep alive flag +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_load_conf(const char *filename, GroupArray *pGroupArray, \ + bool *bKeepAlive); + + +int fdht_connect_all_servers(GroupArray *pGroupArray, const bool bKeepAlive, \ + int *success_count, int *fail_count); + +void fdht_disconnect_all_servers(GroupArray *pGroupArray); + +/* +destroy function, free resource +param: +return: none +*/ +void fdht_client_destroy(); + +#define fdht_get(pKeyInfo, ppValue, value_len) \ + fdht_get_ex1((&g_group_array), g_keep_alive, pKeyInfo, \ + FDHT_EXPIRES_NONE, ppValue, value_len, malloc) + +#define fdht_get_ex(pKeyInfo, expires, ppValue, value_len) \ + fdht_get_ex1((&g_group_array), g_keep_alive, pKeyInfo, expires, \ + ppValue, value_len, malloc) + +#define fdht_batch_get(pObjectInfo, key_list, key_count, success_count) \ + fdht_batch_get_ex1((&g_group_array), g_keep_alive, pObjectInfo, \ + key_list, key_count, FDHT_EXPIRES_NONE, \ + malloc, success_count) + +#define fdht_batch_get_ex(pObjectInfo, key_list, key_count, \ + expires, success_count) \ + fdht_batch_get_ex1((&g_group_array), g_keep_alive, pObjectInfo, \ + key_list, key_count, expires, malloc, success_count) + +#define fdht_set(pKeyInfo, expires, pValue, value_len) \ + fdht_set_ex((&g_group_array), g_keep_alive, pKeyInfo, expires, \ + pValue, value_len) + +#define fdht_batch_set(pObjectInfo, key_list, key_count, \ + expires, success_count) \ + fdht_batch_set_ex((&g_group_array), g_keep_alive, pObjectInfo, \ + key_list, key_count, expires, success_count) + +#define fdht_inc(pKeyInfo, expires, increase, pValue, value_len) \ + fdht_inc_ex((&g_group_array), g_keep_alive, pKeyInfo, expires, \ + increase, pValue, value_len) + +#define fdht_delete(pKeyInfo) \ + fdht_delete_ex((&g_group_array), g_keep_alive, pKeyInfo) + +#define fdht_batch_delete(pObjectInfo, key_list, key_count, success_count) \ + fdht_batch_delete_ex((&g_group_array), g_keep_alive, pObjectInfo, \ + key_list, key_count, success_count) + +#define fdht_stat(server_index, buff, size) \ + fdht_stat_ex((&g_group_array), g_keep_alive, server_index, buff, size) + +/* +get value of the key +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pKeyInfo: the key to fetch + expires: expire time (unix timestamp) + FDHT_EXPIRES_NONE - do not change the expire time of the key + FDHT_EXPIRES_NEVER- set the expire time to forever(never expired) + ppValue: return the value of the key + value_len: return the length of the value (bytes) + malloc_func: malloc function, can be standard function named malloc +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_get_ex1(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo, const time_t expires, \ + char **ppValue, int *value_len, MallocFunc malloc_func); + +/* +get values of the key list +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pObjectInfo: the object to fetch, namespace and object id can't be empty + key_list: key list, return the value of the key + key_count: key count + expires: expire time (unix timestamp) + FDHT_EXPIRES_NONE - do not change the expire time of the keys + FDHT_EXPIRES_NEVER- set the expire time to forever(never expired) + malloc_func: malloc function, can be standard function named malloc +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_batch_get_ex1(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTObjectInfo *pObjectInfo, FDHTKeyValuePair *key_list, \ + const int key_count, const time_t expires, \ + MallocFunc malloc_func, int *success_count); + +/* +set value of the key +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pKeyInfo: the key to set + expires: expire time (unix timestamp) + FDHT_EXPIRES_NEVER- set the expire time to forever(never expired) + pValue: the value of the key + value_len: the length of the value (bytes) +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_set_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo, const time_t expires, \ + const char *pValue, const int value_len); + +/* +set values of the key list +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pObjectInfo: the object to fetch, namespace and object id can't be empty + key_list: key list, return the value of the key + key_count: key count + expires: expire time (unix timestamp) + FDHT_EXPIRES_NEVER- set the expire time to forever(never expired) +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_batch_set_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTObjectInfo *pObjectInfo, FDHTKeyValuePair *key_list, \ + const int key_count, const time_t expires, int *success_count); + +/* +increase value of the key, if the key does not exist, +set the value to increment value (param named "increase") +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pKeyInfo: the key to increase + expires: expire time (unix timestamp) + FDHT_EXPIRES_NEVER- set the expire time to forever(never expired) + increase: the increment value, can be negative, eg. 1 or -1 + pValue: return the value after increment + value_len: return the length of the value (bytes) +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_inc_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo, const time_t expires, \ + const int increase, char *pValue, int *value_len); + +/* +delete the key +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pKeyInfo: the key to delete +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_delete_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTKeyInfo *pKeyInfo); + +/* +delete key list +param: + pGroupArray: group info, can use &g_group_array + bKeepAlive: persistent connection flag, true for persistent connection + pObjectInfo: the object to fetch, namespace and object id can't be empty + key_list: key list, return the value of the key + key_count: key count +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_batch_delete_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + FDHTObjectInfo *pObjectInfo, FDHTKeyValuePair *key_list, \ + const int key_count, int *success_count); + +/* +query stat of server +param: + pGroupArray: group info, can use &g_group_array + server_index: index of server, based 0 + buff: return stat buff, key value pair as key=value, row seperated by \n + size: buff size +return: 0 for success, != 0 for fail (errno) +*/ +int fdht_stat_ex(GroupArray *pGroupArray, const bool bKeepAlive, \ + const int server_index, char *buff, const int size); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/fdht_client/fdht_define.h b/storage/fdht_client/fdht_define.h new file mode 100644 index 0000000..cc4c724 --- /dev/null +++ b/storage/fdht_client/fdht_define.h @@ -0,0 +1,47 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_define.h + +#ifndef _FDHT_DEFINE_H_ +#define _FDHT_DEFINE_H_ + +#include "common_define.h" + +#define FDHT_SERVER_DEFAULT_PORT 24000 +#define FDHT_DEFAULT_PROXY_PORT 12200 +#define FDHT_MAX_PKG_SIZE 64 * 1024 +#define FDHT_MIN_BUFF_SIZE 64 * 1024 +#define FDHT_DEFAULT_MAX_THREADS 64 +#define DEFAULT_SYNC_DB_INVERVAL 86400 +#define DEFAULT_SYNC_WAIT_MSEC 100 +#define DEFAULT_CLEAR_EXPIRED_INVERVAL 0 +#define DEFAULT_DB_DEAD_LOCK_DETECT_INVERVAL 1000 +#define FDHT_MAX_KEY_COUNT_PER_REQ 128 +#define SYNC_BINLOG_BUFF_DEF_INTERVAL 60 +#define COMPRESS_BINLOG_DEF_INTERVAL 86400 +#define DEFAULT_SYNC_STAT_FILE_INTERVAL 300 +#define FDHT_DEFAULT_SYNC_MARK_FILE_FREQ 5000 + +#define FDHT_STORE_TYPE_BDB 1 +#define FDHT_STORE_TYPE_MPOOL 2 + +#define FDHT_DEFAULT_MPOOL_INIT_CAPACITY 10000 +#define FDHT_DEFAULT_MPOOL_LOAD_FACTOR 0.75 +#define FDHT_DEFAULT_MPOOL_CLEAR_MIN_INTEVAL 300 + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/fdht_client/fdht_func.c b/storage/fdht_client/fdht_func.c new file mode 100644 index 0000000..be8c33f --- /dev/null +++ b/storage/fdht_client/fdht_func.c @@ -0,0 +1,765 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_func.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "sockopt.h" +#include "shared_func.h" +#include "ini_file_reader.h" +#include "fdht_func.h" + +int fdht_split_ids(const char *szIds, int **ppIds, int *id_count) +{ + char *pBuff; + char *pNumStart; + char *p; + int alloc_count; + int result; + int count; + int i; + char ch; + char *pNumStart1; + char *pNumStart2; + int nNumLen1; + int nNumLen2; + int nStart; + int nEnd; + + alloc_count = getOccurCount(szIds, ',') + 1; + *id_count = 0; + *ppIds = (int *)malloc(sizeof(int) * alloc_count); + if (*ppIds == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s.", \ + __LINE__, (int)sizeof(int) * alloc_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + pBuff = strdup(szIds); + if (pBuff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "strdup \"%s\" fail, errno: %d, error info: %s.", \ + __LINE__, szIds, \ + errno, STRERROR(errno)); + free(*ppIds); + *ppIds = NULL; + return errno != 0 ? errno : ENOMEM; + } + + result = 0; + p = pBuff; + while (*p != '\0') + { + while (*p == ' ' || *p == '\t') //trim prior spaces + { + p++; + } + + if (*p == '\0') + { + break; + } + + if (*p >= '0' && *p <= '9') + { + pNumStart = p; + p++; + while (*p >= '0' && *p <= '9') + { + p++; + } + + if (p - pNumStart == 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group ids \"%s\", " \ + "which contains empty group id!", \ + __LINE__, szIds); + result = EINVAL; + break; + } + + ch = *p; + if (!(ch == ',' || ch == '\0')) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group ids \"%s\", which contains " \ + "invalid char: %c(0x%02X)! remain string: %s", \ + __LINE__, szIds, *p, *p, p); + result = EINVAL; + break; + } + + *p = '\0'; + (*ppIds)[*id_count] = atoi(pNumStart); + (*id_count)++; + + if (ch == '\0') + { + break; + } + + p++; //skip , + continue; + } + + if (*p != '[') + { + logError("file: "__FILE__", line: %d, " \ + "invalid group ids \"%s\", which contains " \ + "invalid char: %c(0x%02X)! remain string: %s", \ + __LINE__, szIds, *p, *p, p); + result = EINVAL; + break; + } + + p++; //skip [ + while (*p == ' ' || *p == '\t') //trim prior spaces + { + p++; + } + + pNumStart1 = p; + while (*p >='0' && *p <= '9') + { + p++; + } + + nNumLen1 = p - pNumStart1; + if (nNumLen1 == 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group ids: %s, " \ + "empty entry before char %c(0x%02X), " \ + "remain string: %s", \ + __LINE__, szIds, *p, *p, p); + result = EINVAL; + break; + } + + while (*p == ' ' || *p == '\t') //trim spaces + { + p++; + } + + if (*p != '-') + { + logError("file: "__FILE__", line: %d, " \ + "expect \"-\", but char %c(0x%02X) ocurs " \ + "in group ids: %s, remain string: %s",\ + __LINE__, *p, *p, szIds, p); + result = EINVAL; + break; + } + + *(pNumStart1 + nNumLen1) = '\0'; + nStart = atoi(pNumStart1); + + p++; //skip - + while (*p == ' ' || *p == '\t') //trim spaces + { + p++; + } + + pNumStart2 = p; + while (*p >='0' && *p <= '9') + { + p++; + } + + nNumLen2 = p - pNumStart2; + if (nNumLen2 == 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group ids: %s, " \ + "empty entry before char %c(0x%02X)", \ + __LINE__, szIds, *p, *p); + result = EINVAL; + break; + } + + /* trim tail spaces */ + while (*p == ' ' || *p == '\t') + { + p++; + } + + if (*p != ']') + { + logError("file: "__FILE__", line: %d, " \ + "expect \"]\", but char %c(0x%02X) ocurs " \ + "in group ids: %s",\ + __LINE__, *p, *p, szIds); + result = EINVAL; + break; + } + + *(pNumStart2 + nNumLen2) = '\0'; + nEnd = atoi(pNumStart2); + + count = nEnd - nStart; + if (count < 0) + { + count = 0; + } + if (alloc_count < *id_count + (count + 1)) + { + alloc_count += count + 1; + *ppIds = (int *)realloc(*ppIds, \ + sizeof(int) * alloc_count); + if (*ppIds == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, "\ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, \ + (int)sizeof(int) * alloc_count,\ + result, STRERROR(result)); + + break; + } + } + + for (i=nStart; i<=nEnd; i++) + { + (*ppIds)[*id_count] = i; + (*id_count)++; + } + + p++; //skip ] + /* trim spaces */ + while (*p == ' ' || *p == '\t') + { + p++; + } + if (*p == ',') + { + p++; //skip , + } + } + + free(pBuff); + + if (result == 0 && *id_count == 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group ids count: 0!", __LINE__); + result = EINVAL; + } + + if (result != 0) + { + *id_count = 0; + free(*ppIds); + *ppIds = NULL; + } + + printf("*id_count=%d\n", *id_count); + for (i=0; i<*id_count; i++) + { + printf("%d\n", (*ppIds)[i]); + } + + return result; +} + +static int fdht_cmp_by_ip_and_port_p(const void *p1, const void *p2) +{ + int res; + + res = strcmp(((FDHTServerInfo*)p1)->ip_addr, \ + ((FDHTServerInfo*)p2)->ip_addr); + if (res != 0) + { + return res; + } + + return ((FDHTServerInfo*)p1)->port - \ + ((FDHTServerInfo*)p2)->port; +} + +static int fdht_cmp_by_ip_and_port_pp(const void *p1, const void *p2) +{ + int res; + + res = strcmp((*((FDHTServerInfo **)p1))->ip_addr, \ + (*((FDHTServerInfo **)p2))->ip_addr); + if (res != 0) + { + return res; + } + + return ((*(FDHTServerInfo **)p1))->port - \ + (*((FDHTServerInfo **)p2))->port; +} + +static void fdht_insert_sorted_servers(GroupArray *pGroupArray, \ + FDHTServerInfo *pInsertedServer) +{ + FDHTServerInfo *pCurrent; + + for (pCurrent=pGroupArray->servers + pGroupArray->server_count; \ + pCurrent > pGroupArray->servers; pCurrent--) + { + if (fdht_cmp_by_ip_and_port_p(pCurrent-1, pInsertedServer)<0) + { + break; + } + + memcpy(pCurrent, pCurrent - 1, sizeof(FDHTServerInfo)); + } + + memcpy(pCurrent, pInsertedServer, sizeof(FDHTServerInfo)); +} + +int fdht_load_groups_ex(IniContext *pIniContext, \ + GroupArray *pGroupArray, const bool bLoadProxyParams) +{ + IniItem *pItemInfo; + IniItem *pItemEnd; + int group_id; + char item_name[32]; + ServerArray *pServerArray; + FDHTServerInfo **ppServer; + FDHTServerInfo **ppServerEnd; + FDHTServerInfo **ppServers; + FDHTServerInfo *pServerInfo; + FDHTServerInfo *pServerEnd; + FDHTServerInfo *pFound; + int alloc_server_count; + char *ip_port[2]; + char *pProxyIpAddr; + + pGroupArray->group_count = iniGetIntValue(NULL, "group_count", \ + pIniContext, 0); + if (pGroupArray->group_count <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group count: %d <= 0!", \ + __LINE__, pGroupArray->group_count); + return EINVAL; + } + + pGroupArray->groups = (ServerArray *)malloc(sizeof(ServerArray) * \ + pGroupArray->group_count); + if (pGroupArray->groups == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(ServerArray) * pGroupArray->group_count,\ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + ppServers = (FDHTServerInfo **)malloc(sizeof(FDHTServerInfo *) * \ + pGroupArray->group_count); + if (ppServers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDHTServerInfo *) * \ + pGroupArray->group_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + alloc_server_count = pGroupArray->group_count * 2; + pGroupArray->servers = (FDHTServerInfo *)malloc(sizeof(FDHTServerInfo) \ + * alloc_server_count); + if (pGroupArray->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(FDHTServerInfo) * alloc_server_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + pGroupArray->server_count = 0; + pServerArray = pGroupArray->groups; + for (group_id=0; group_idgroup_count; group_id++) + { + sprintf(item_name, "group%d", group_id); + pItemInfo = iniGetValuesEx(NULL, item_name, pIniContext, \ + &(pServerArray->count)); + if (pItemInfo == NULL || pServerArray->count <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "group %d not exist!", __LINE__, group_id); + return ENOENT; + } + + pServerArray->servers = (FDHTServerInfo **)malloc( \ + sizeof(FDHTServerInfo *) * pServerArray->count); + if (pServerArray->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(FDHTServerInfo)*pServerArray->count, + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + ppServers[group_id] = (FDHTServerInfo *)malloc( \ + sizeof(FDHTServerInfo) * pServerArray->count); + if (ppServers[group_id] == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(FDHTServerInfo *)*pServerArray->count, + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(ppServers[group_id], 0, sizeof(FDHTServerInfo) * \ + pServerArray->count); + + ppServer = pServerArray->servers; + pServerInfo = ppServers[group_id]; + pItemEnd = pItemInfo + pServerArray->count; + for (; pItemInfovalue, ':', ip_port, 2) != 2) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" 's value \"%s\" is invalid, "\ + "correct format is hostname:port", \ + __LINE__, item_name, pItemInfo->value); + return EINVAL; + } + + if (getIpaddrByName(ip_port[0], pServerInfo->ip_addr, \ + sizeof(pServerInfo->ip_addr)) == INADDR_NONE) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" 's value \"%s\" is invalid, "\ + "invalid hostname: %s", \ + __LINE__, item_name, \ + pItemInfo->value, ip_port[0]); + return EINVAL; + } + + if (strcmp(pServerInfo->ip_addr, "127.0.0.1") == 0) + { + logError("file: "__FILE__", line: %d, " \ + "group%d: invalid hostname \"%s\", " \ + "ip address can not be 127.0.0.1!", \ + __LINE__, group_id, pItemInfo->value); + return EINVAL; + } + + pServerInfo->port = atoi(ip_port[1]); + if (pServerInfo->port <= 0 || pServerInfo->port > 65535) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" 's value \"%s\" is invalid, "\ + "invalid port: %d", \ + __LINE__, item_name, \ + pItemInfo->value, pServerInfo->port); + return EINVAL; + } + + pServerInfo->sock = -1; + pFound = (FDHTServerInfo *)bsearch(pServerInfo, \ + pGroupArray->servers, \ + pGroupArray->server_count, \ + sizeof(FDHTServerInfo), \ + fdht_cmp_by_ip_and_port_p); + if (pFound == NULL) //not found + { + if (pGroupArray->server_count >= \ + alloc_server_count) + { + alloc_server_count = \ + pGroupArray->server_count + \ + pGroupArray->group_count + 8; + pGroupArray->servers = (FDHTServerInfo*) + realloc(pGroupArray->servers, \ + sizeof(FDHTServerInfo) * \ + alloc_server_count); + if (pGroupArray->servers == NULL) + { + logError("file: "__FILE__", " \ + "line: %d, malloc " \ + "%d bytes fail, " + "errno: %d, " \ + "error info: %s", \ + __LINE__, \ + (int)sizeof(FDHTServerInfo) \ + * alloc_server_count, \ + errno, STRERROR(errno)); + return errno!=0 ? errno:ENOMEM; + } + } + + fdht_insert_sorted_servers( \ + pGroupArray, pServerInfo); + pGroupArray->server_count++; + } + + ppServer++; + pServerInfo++; + } + + pServerArray++; + } + + pServerArray = pGroupArray->groups; + for (group_id=0; group_idgroup_count; group_id++) + { + ppServer = pServerArray->servers; + pServerEnd = ppServers[group_id] + pServerArray->count; + for (pServerInfo=ppServers[group_id]; \ + pServerInfoservers, \ + pGroupArray->server_count, \ + sizeof(FDHTServerInfo), \ + fdht_cmp_by_ip_and_port_p); + ppServer++; + } + + qsort(pServerArray->servers, pServerArray->count, \ + sizeof(FDHTServerInfo *), fdht_cmp_by_ip_and_port_pp); + ppServerEnd = pServerArray->servers + pServerArray->count; + for (ppServer=pServerArray->servers + 1; \ + ppServerip_addr, \ + (*ppServer)->port); + return EINVAL; + } + } + + pServerArray++; + } + + for (group_id=0; group_idgroup_count; group_id++) + { + free(ppServers[group_id]); + } + free(ppServers); + + if (alloc_server_count > pGroupArray->server_count) + { + pGroupArray->servers = (FDHTServerInfo*)realloc( \ + pGroupArray->servers, sizeof(FDHTServerInfo) \ + * pGroupArray->server_count); + if (pGroupArray->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDHTServerInfo) * \ + pGroupArray->server_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + } + + memset(&pGroupArray->proxy_server, 0, sizeof(FDHTServerInfo)); + if (!bLoadProxyParams) + { + return 0; + } + + pGroupArray->use_proxy = iniGetBoolValue(NULL, "use_proxy", \ + pIniContext, false); + if (!pGroupArray->use_proxy) + { + return 0; + } + + pProxyIpAddr = iniGetStrValue(NULL, "proxy_addr", \ + pIniContext); + if (pProxyIpAddr == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "item \"proxy_addr\" not exists!", \ + __LINE__); + return ENOENT; + } + snprintf(pGroupArray->proxy_server.ip_addr, \ + sizeof(pGroupArray->proxy_server.ip_addr), \ + "%s", pProxyIpAddr); + + pGroupArray->proxy_server.port = iniGetIntValue(NULL, "proxy_port", \ + pIniContext, FDHT_DEFAULT_PROXY_PORT); + if (pGroupArray->proxy_server.port <= 0 || \ + pGroupArray->proxy_server.port > 65535) + { + logError("file: "__FILE__", line: %d, " \ + "proxy_port: %d is invalid!", \ + __LINE__, pGroupArray->proxy_server.port); + return EINVAL; + } + + pGroupArray->proxy_server.sock = -1; + + return 0; +} + +int fdht_copy_group_array(GroupArray *pDestGroupArray, \ + GroupArray *pSrcGroupArray) +{ + ServerArray *pSrcArray; + ServerArray *pServerArray; + ServerArray *pArrayEnd; + FDHTServerInfo *pServerInfo; + FDHTServerInfo *pServerEnd; + FDHTServerInfo **ppSrcServer; + FDHTServerInfo **ppServerInfo; + FDHTServerInfo **ppServerEnd; + + memcpy(pDestGroupArray, pSrcGroupArray, sizeof(GroupArray)); + pDestGroupArray->groups = (ServerArray *)malloc(sizeof(ServerArray) * \ + pDestGroupArray->group_count); + if (pDestGroupArray->groups == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)sizeof(ServerArray) * \ + pDestGroupArray->group_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + pDestGroupArray->servers = (FDHTServerInfo *)malloc( \ + sizeof(FDHTServerInfo) * pDestGroupArray->server_count); + if (pDestGroupArray->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDHTServerInfo) * \ + pDestGroupArray->server_count, \ + errno, STRERROR(errno)); + + free(pDestGroupArray->groups); + pDestGroupArray->groups = NULL; + return errno != 0 ? errno : ENOMEM; + } + memcpy(pDestGroupArray->servers, pSrcGroupArray->servers, \ + sizeof(FDHTServerInfo) * pDestGroupArray->server_count); + + pSrcArray = pSrcGroupArray->groups; + pArrayEnd = pDestGroupArray->groups + pDestGroupArray->group_count; + for (pServerArray=pDestGroupArray->groups; pServerArraycount = pSrcArray->count; + pServerArray->servers = (FDHTServerInfo **)malloc( \ + sizeof(FDHTServerInfo *) * \ + pServerArray->count); + if (pServerArray->servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDHTServerInfo *) * \ + pServerArray->count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + ppSrcServer = pSrcArray->servers; + ppServerEnd = pServerArray->servers + pServerArray->count; + for (ppServerInfo=pServerArray->servers; \ + ppServerInfoservers + \ + (*ppSrcServer - pSrcGroupArray->servers); + ppSrcServer++; + } + + pSrcArray++; + } + + pServerEnd = pDestGroupArray->servers + pDestGroupArray->server_count; + for (pServerInfo=pDestGroupArray->servers; \ + pServerInfosock >= 0) + { + pServerInfo->sock = -1; + } + } + + return 0; +} + +void fdht_free_group_array(GroupArray *pGroupArray) +{ + ServerArray *pServerArray; + ServerArray *pArrayEnd; + FDHTServerInfo *pServerInfo; + FDHTServerInfo *pServerEnd; + + if (pGroupArray->servers != NULL) + { + pArrayEnd = pGroupArray->groups + pGroupArray->group_count; + for (pServerArray=pGroupArray->groups; pServerArrayservers == NULL) + { + continue; + } + + free(pServerArray->servers); + pServerArray->servers = NULL; + } + + pServerEnd = pGroupArray->servers + pGroupArray->server_count; + for (pServerInfo=pGroupArray->servers; \ + pServerInfosock >= 0) + { + close(pServerInfo->sock); + pServerInfo->sock = -1; + } + } + + free(pGroupArray->servers); + pGroupArray->servers = NULL; + } + + if (pGroupArray->groups != NULL) + { + free(pGroupArray->groups); + pGroupArray->groups = NULL; + } +} + diff --git a/storage/fdht_client/fdht_func.h b/storage/fdht_client/fdht_func.h new file mode 100644 index 0000000..af0e036 --- /dev/null +++ b/storage/fdht_client/fdht_func.h @@ -0,0 +1,50 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_func.h + +#ifndef _FDHT_FUNC_H_ +#define _FDHT_FUNC_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdht_types.h" +#include "ini_file_reader.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int fdht_split_ids(const char *szIds, int **ppIds, int *id_count); + +#define fdht_load_groups(pIniContext, pGroupArray) \ + fdht_load_groups_ex(pIniContext, pGroupArray, true) + +int fdht_load_groups_ex(IniContext *pIniContext, \ + GroupArray *pGroupArray, const bool bLoadProxyParams); + +int fdht_copy_group_array(GroupArray *pDestGroupArray, \ + GroupArray *pSrcGroupArray); +void fdht_free_group_array(GroupArray *pGroupArray); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/fdht_client/fdht_global.c b/storage/fdht_client/fdht_global.c new file mode 100644 index 0000000..ce93a63 --- /dev/null +++ b/storage/fdht_client/fdht_global.c @@ -0,0 +1,18 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include "fdht_global.h" + +int g_fdht_connect_timeout = DEFAULT_CONNECT_TIMEOUT; +int g_fdht_network_timeout = DEFAULT_NETWORK_TIMEOUT; +char g_fdht_base_path[MAX_PATH_SIZE] = {'/', 't', 'm', 'p', '\0'}; +Version g_fdht_version = {1, 14}; + diff --git a/storage/fdht_client/fdht_global.h b/storage/fdht_client/fdht_global.h new file mode 100644 index 0000000..20c6cd1 --- /dev/null +++ b/storage/fdht_client/fdht_global.h @@ -0,0 +1,37 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_global.h + +#ifndef _FDHT_GLOBAL_H +#define _FDHT_GLOBAL_H + +#include +#include +#include +#include +#include +#include "common_define.h" +#include "fdht_define.h" +#include "fdht_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_fdht_connect_timeout; +extern int g_fdht_network_timeout; +extern char g_fdht_base_path[MAX_PATH_SIZE]; +extern Version g_fdht_version; + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/fdht_client/fdht_proto.c b/storage/fdht_client/fdht_proto.c new file mode 100644 index 0000000..6cef30a --- /dev/null +++ b/storage/fdht_client/fdht_proto.c @@ -0,0 +1,470 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdht_define.h" +#include "shared_func.h" +#include "logger.h" +#include "sockopt.h" +#include "fdht_types.h" +#include "fdht_proto.h" + +extern int g_fdht_network_timeout; + +int fdht_recv_header(FDHTServerInfo *pServer, fdht_pkg_size_t *in_bytes) +{ + FDHTProtoHeader resp; + int result; + + if ((result=tcprecvdata_nb(pServer->sock, &resp, \ + sizeof(resp), g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, \ + pServer->port, \ + result, STRERROR(result)); + *in_bytes = 0; + return result; + } + + if (resp.status != 0) + { + *in_bytes = 0; + return resp.status; + } + + *in_bytes = buff2int(resp.pkg_len); + if (*in_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv package size %d " \ + "is not correct", \ + __LINE__, pServer->ip_addr, \ + pServer->port, *in_bytes); + *in_bytes = 0; + return EINVAL; + } + + return resp.status; +} + +int fdht_recv_response(FDHTServerInfo *pServer, \ + char **buff, const int buff_size, \ + fdht_pkg_size_t *in_bytes) +{ + int result; + bool bMalloced; + + result = fdht_recv_header(pServer, in_bytes); + if (result != 0) + { + return result; + } + + if (*in_bytes == 0) + { + return 0; + } + + if (*buff == NULL) + { + *buff = (char *)malloc((*in_bytes) + 1); + if (*buff == NULL) + { + *in_bytes = 0; + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (*in_bytes) + 1); + return errno != 0 ? errno : ENOMEM; + } + + bMalloced = true; + } + else + { + if (*in_bytes > buff_size) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv body bytes: %d" \ + " exceed max: %d", \ + __LINE__, pServer->ip_addr, \ + pServer->port, *in_bytes, buff_size); + *in_bytes = 0; + return ENOSPC; + } + + bMalloced = false; + } + + if ((result=tcprecvdata_nb(pServer->sock, *buff, \ + *in_bytes, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, \ + pServer->port, \ + result, STRERROR(result)); + *in_bytes = 0; + if (bMalloced) + { + free(*buff); + *buff = NULL; + } + return result; + } + + return 0; +} + +int fdht_quit(FDHTServerInfo *pServer) +{ + FDHTProtoHeader header; + int result; + + memset(&header, 0, sizeof(header)); + header.cmd = FDHT_PROTO_CMD_QUIT; + result = tcpsenddata_nb(pServer->sock, &header, sizeof(header), \ + g_fdht_network_timeout); + if(result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server ip: %s, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pServer->ip_addr, \ + result, STRERROR(result)); + return result; + } + + return 0; +} + +void fdht_disconnect_server(FDHTServerInfo *pServer) +{ + if (pServer->sock > 0) + { + close(pServer->sock); + pServer->sock = -1; + } +} + +int fdht_connect_server_nb(FDHTServerInfo *pServer, const int connect_timeout) +{ + int result; + + if (pServer->sock > 0) + { + close(pServer->sock); + } + pServer->sock = socket(AF_INET, SOCK_STREAM, 0); + if(pServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s", __LINE__, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + if ((result=tcpsetnonblockopt(pServer->sock)) != 0) + { + close(pServer->sock); + pServer->sock = -1; + return result; + } + + if ((result=connectserverbyip_nb(pServer->sock, \ + pServer->ip_addr, pServer->port, connect_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to %s:%d fail, errno: %d, " \ + "error info: %s", __LINE__, pServer->ip_addr, \ + pServer->port, result, STRERROR(result)); + + close(pServer->sock); + pServer->sock = -1; + return result; + } + + return 0; +} + +int fdht_connect_server(FDHTServerInfo *pServer) +{ + int result; + + if (pServer->sock > 0) + { + close(pServer->sock); + } + pServer->sock = socket(AF_INET, SOCK_STREAM, 0); + if(pServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s", __LINE__, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + if ((result=connectserverbyip(pServer->sock, \ + pServer->ip_addr, pServer->port)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to %s:%d fail, errno: %d, " \ + "error info: %s", __LINE__, pServer->ip_addr, \ + pServer->port, result, STRERROR(result)); + + close(pServer->sock); + pServer->sock = -1; + return result; + } + + if ((result=tcpsetnonblockopt(pServer->sock)) != 0) + { + close(pServer->sock); + pServer->sock = -1; + return result; + } + + return 0; +} + +/** +* request body format: +* namespace_len: 4 bytes big endian integer +* namespace: can be emtpy +* obj_id_len: 4 bytes big endian integer +* object_id: the object id (can be empty) +* key_len: 4 bytes big endian integer +* key: key name +* value_len: 4 bytes big endian integer +* value: value buff +* response body format: +* none +*/ +int fdht_client_set(FDHTServerInfo *pServer, const char keep_alive, \ + const time_t timestamp, const time_t expires, const int prot_cmd, \ + const int key_hash_code, FDHTKeyInfo *pKeyInfo, \ + const char *pValue, const int value_len) +{ + int result; + char buff[sizeof(FDHTProtoHeader) + FDHT_MAX_FULL_KEY_LEN + 16 + 1024]; + FDHTProtoHeader *pHeader; + int in_bytes; + char *p; + + memset(buff, 0, sizeof(buff)); + pHeader = (FDHTProtoHeader *)buff; + pHeader->cmd = prot_cmd; + pHeader->keep_alive = keep_alive; + int2buff((int)timestamp, pHeader->timestamp); + int2buff((int)expires, pHeader->expires); + int2buff(key_hash_code, pHeader->key_hash_code); + int2buff(16 + pKeyInfo->namespace_len + pKeyInfo->obj_id_len + \ + pKeyInfo->key_len + value_len, pHeader->pkg_len); + + p = buff + sizeof(FDHTProtoHeader); + PACK_BODY_UNTIL_KEY(pKeyInfo, p) + int2buff(value_len, p); + p += 4; + + if ((p - buff) + value_len <= sizeof(buff)) + { + memcpy(p, pValue, value_len); + p += value_len; + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + } + else + { + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=tcpsenddata_nb(pServer->sock, (char *)pValue, \ + value_len, g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "recv data from server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server %s:%d reponse bytes: %d != 0", \ + __LINE__, pServer->ip_addr, \ + pServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +/** +* request body format: +* namespace_len: 4 bytes big endian integer +* namespace: can be emtpy +* obj_id_len: 4 bytes big endian integer +* object_id: the object id (can be empty) +* key_len: 4 bytes big endian integer +* key: key name +* response body format: +* none +*/ +int fdht_client_delete(FDHTServerInfo *pServer, const char keep_alive, \ + const time_t timestamp, const int prot_cmd, \ + const int key_hash_code, FDHTKeyInfo *pKeyInfo) +{ + int result; + FDHTProtoHeader *pHeader; + char buff[sizeof(FDHTProtoHeader) + FDHT_MAX_FULL_KEY_LEN + 16]; + int in_bytes; + char *p; + + memset(buff, 0, sizeof(buff)); + pHeader = (FDHTProtoHeader *)buff; + pHeader->cmd = prot_cmd; + pHeader->keep_alive = keep_alive; + int2buff(timestamp, pHeader->timestamp); + int2buff(key_hash_code, pHeader->key_hash_code); + int2buff(12 + pKeyInfo->namespace_len + pKeyInfo->obj_id_len + \ + pKeyInfo->key_len, pHeader->pkg_len); + + p = buff + sizeof(FDHTProtoHeader); + PACK_BODY_UNTIL_KEY(pKeyInfo, p) + + if ((result=tcpsenddata_nb(pServer->sock, buff, p - buff, \ + g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "recv data from server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "recv data from server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + } + + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server %s:%d reponse bytes: %d != 0", \ + __LINE__, pServer->ip_addr, \ + pServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +int fdht_client_heart_beat(FDHTServerInfo *pServer) +{ + int result; + FDHTProtoHeader header; + int in_bytes; + + memset(&header, 0, sizeof(header)); + header.cmd = FDHT_PROTO_CMD_HEART_BEAT; + header.keep_alive = 1; + + if ((result=tcpsenddata_nb(pServer->sock, &header, \ + sizeof(header), g_fdht_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=fdht_recv_header(pServer, &in_bytes)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "recv data from server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pServer->ip_addr, pServer->port, \ + result, STRERROR(result)); + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server %s:%d reponse bytes: %d != 0", \ + __LINE__, pServer->ip_addr, \ + pServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + diff --git a/storage/fdht_client/fdht_proto.h b/storage/fdht_client/fdht_proto.h new file mode 100644 index 0000000..93e87b9 --- /dev/null +++ b/storage/fdht_client/fdht_proto.h @@ -0,0 +1,69 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_proto.h + +#ifndef _FDHT_PROTO_H_ +#define _FDHT_PROTO_H_ + +#include "fdht_types.h" +#include "fdht_proto_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int fdht_recv_header(FDHTServerInfo *pServer, fdht_pkg_size_t *in_bytes); + +int fdht_recv_response(FDHTServerInfo *pServer, \ + char **buff, const int buff_size, \ + fdht_pkg_size_t *in_bytes); +int fdht_quit(FDHTServerInfo *pServer); + +/** +* connect to the server (block mode) +* params: +* pServer: server +* return: 0 success, !=0 fail, return the error code +**/ +int fdht_connect_server(FDHTServerInfo *pServer); + +/** +* connect to the server (non-block mode) +* params: +* pServer: server +* return: 0 success, !=0 fail, return the error code +**/ +int fdht_connect_server_nb(FDHTServerInfo *pServer, const int connect_timeout); + +/** +* close connection to the server +* params: +* pServer: server +* return: +**/ +void fdht_disconnect_server(FDHTServerInfo *pServer); + + +int fdht_client_set(FDHTServerInfo *pServer, const char keep_alive, \ + const time_t timestamp, const time_t expires, const int prot_cmd, \ + const int key_hash_code, FDHTKeyInfo *pKeyInfo, \ + const char *pValue, const int value_len); + +int fdht_client_delete(FDHTServerInfo *pServer, const char keep_alive, \ + const time_t timestamp, const int prot_cmd, \ + const int key_hash_code, FDHTKeyInfo *pKeyInfo); + +int fdht_client_heart_beat(FDHTServerInfo *pServer); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/fdht_client/fdht_proto_types.h b/storage/fdht_client/fdht_proto_types.h new file mode 100644 index 0000000..cdf2588 --- /dev/null +++ b/storage/fdht_client/fdht_proto_types.h @@ -0,0 +1,85 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_proto_types.h + +#ifndef _FDHT_PROTO_TYPES_H_ +#define _FDHT_PROTO_TYPES_H_ + +#define FDHT_PROTO_CMD_QUIT 10 + +#define FDHT_PROTO_CMD_SET 11 +#define FDHT_PROTO_CMD_INC 12 +#define FDHT_PROTO_CMD_GET 13 +#define FDHT_PROTO_CMD_DEL 14 +#define FDHT_PROTO_CMD_BATCH_SET 15 +#define FDHT_PROTO_CMD_BATCH_GET 16 +#define FDHT_PROTO_CMD_BATCH_DEL 17 +#define FDHT_PROTO_CMD_STAT 18 + +#define FDHT_PROTO_CMD_SYNC_REQ 21 +#define FDHT_PROTO_CMD_SYNC_NOTIFY 22 //sync done notify +#define FDHT_PROTO_CMD_SYNC_SET 23 +#define FDHT_PROTO_CMD_SYNC_DEL 24 + +#define FDHT_PROTO_CMD_HEART_BEAT 30 + +#define FDHT_PROTO_CMD_RESP 40 + +#define FDHT_PROTO_PKG_LEN_SIZE 4 +#define FDHT_PROTO_CMD_SIZE 1 + +typedef int fdht_pkg_size_t; + +#define PACK_BODY_UNTIL_KEY(pKeyInfo, p) \ + int2buff(pKeyInfo->namespace_len, p); \ + p += 4; \ + if (pKeyInfo->namespace_len > 0) \ + { \ + memcpy(p, pKeyInfo->szNameSpace, pKeyInfo->namespace_len); \ + p += pKeyInfo->namespace_len; \ + } \ + int2buff(pKeyInfo->obj_id_len, p); \ + p += 4; \ + if (pKeyInfo->obj_id_len> 0) \ + { \ + memcpy(p, pKeyInfo->szObjectId, pKeyInfo->obj_id_len); \ + p += pKeyInfo->obj_id_len; \ + } \ + int2buff(pKeyInfo->key_len, p); \ + p += 4; \ + memcpy(p, pKeyInfo->szKey, pKeyInfo->key_len); \ + p += pKeyInfo->key_len; \ + + +#define PACK_BODY_OBJECT(pObjectInfo, p) \ + int2buff(pObjectInfo->namespace_len, p); \ + p += 4; \ + memcpy(p, pObjectInfo->szNameSpace, pObjectInfo->namespace_len); \ + p += pObjectInfo->namespace_len; \ + int2buff(pObjectInfo->obj_id_len, p); \ + p += 4; \ + memcpy(p, pObjectInfo->szObjectId, pObjectInfo->obj_id_len); \ + p += pObjectInfo->obj_id_len; \ + + +typedef struct +{ + char pkg_len[FDHT_PROTO_PKG_LEN_SIZE]; //body length + char key_hash_code[FDHT_PROTO_PKG_LEN_SIZE]; //the key hash code + char timestamp[FDHT_PROTO_PKG_LEN_SIZE]; //current time + + /* key expires, remain timeout = expires - timestamp */ + char expires[FDHT_PROTO_PKG_LEN_SIZE]; + char cmd; + char keep_alive; + char status; +} FDHTProtoHeader; + +#endif + diff --git a/storage/fdht_client/fdht_types.h b/storage/fdht_client/fdht_types.h new file mode 100644 index 0000000..37b0ec1 --- /dev/null +++ b/storage/fdht_client/fdht_types.h @@ -0,0 +1,134 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdht_types.h + +#ifndef _FDHT_TYPES_H +#define _FDHT_TYPES_H + +#include +#include +#include +#include "fdht_define.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define FDHT_MAX_NAMESPACE_LEN 64 +#define FDHT_MAX_OBJECT_ID_LEN 128 +#define FDHT_MAX_SUB_KEY_LEN 128 +#define FDHT_FULL_KEY_SEPERATOR '\x1' + +#define FDHT_EXPIRES_NEVER 0 //never timeout +#define FDHT_EXPIRES_NONE -1 //invalid timeout, should ignore + +#define FDHT_MAX_FULL_KEY_LEN (FDHT_MAX_NAMESPACE_LEN + 1 + \ + FDHT_MAX_OBJECT_ID_LEN + 1 + FDHT_MAX_SUB_KEY_LEN) + +#define FDHT_PACK_FULL_KEY(key_info, full_key, full_key_len, p) \ + p = full_key; \ + if (key_info.namespace_len > 0) \ + { \ + memcpy(p, key_info.szNameSpace, key_info.namespace_len); \ + p += key_info.namespace_len; \ + } \ + *p++ = FDHT_FULL_KEY_SEPERATOR; /*field seperator*/ \ + if (key_info.obj_id_len > 0) \ + { \ + memcpy(p, key_info.szObjectId, key_info.obj_id_len); \ + p += key_info.obj_id_len; \ + } \ + *p++ = FDHT_FULL_KEY_SEPERATOR; /*field seperator*/ \ + memcpy(p, key_info.szKey, key_info.key_len); \ + p += key_info.key_len; \ + full_key_len = p - full_key; + + +typedef struct +{ + int namespace_len; + int obj_id_len; + int key_len; + char szNameSpace[FDHT_MAX_NAMESPACE_LEN + 1]; + char szObjectId[FDHT_MAX_OBJECT_ID_LEN + 1]; + char szKey[FDHT_MAX_SUB_KEY_LEN + 1]; +} FDHTKeyInfo; + +typedef struct +{ + int namespace_len; + int obj_id_len; + char szNameSpace[FDHT_MAX_NAMESPACE_LEN + 1]; + char szObjectId[FDHT_MAX_OBJECT_ID_LEN + 1]; +} FDHTObjectInfo; + +typedef struct +{ + int key_len; + char szKey[FDHT_MAX_SUB_KEY_LEN + 1]; +} FDHTSubKey; + +typedef struct +{ + int key_len; + int value_len; + char szKey[FDHT_MAX_SUB_KEY_LEN + 1]; + char *pValue; + char status; +} FDHTKeyValuePair; + +typedef struct +{ + int sock; + int port; + char ip_addr[IP_ADDRESS_SIZE]; +} FDHTServerInfo; + +typedef struct +{ + char ip_addr[IP_ADDRESS_SIZE]; + bool sync_old_done; + int port; + int sync_req_count; //sync req count + int64_t update_count; //runtime var +} FDHTGroupServer; + +typedef struct { + uint64_t total_set_count; + uint64_t success_set_count; + uint64_t total_inc_count; + uint64_t success_inc_count; + uint64_t total_delete_count; + uint64_t success_delete_count; + uint64_t total_get_count; + uint64_t success_get_count; +} FDHTServerStat; + +typedef struct +{ + FDHTServerInfo **servers; + int count; //server count +} ServerArray; + +typedef struct +{ + ServerArray *groups; + FDHTServerInfo *servers; + int group_count; //group count + int server_count; + FDHTServerInfo proxy_server; + bool use_proxy; +} GroupArray; + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/storage_dio.c b/storage/storage_dio.c new file mode 100644 index 0000000..ea0e675 --- /dev/null +++ b/storage/storage_dio.c @@ -0,0 +1,961 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "pthread_func.h" +#include "logger.h" +#include "sockopt.h" +#include "storage_dio.h" +#include "storage_nio.h" +#include "storage_service.h" +#include "trunk_mem.h" + +static pthread_mutex_t g_dio_thread_lock; +static struct storage_dio_context *g_dio_contexts = NULL; + +int g_dio_thread_count = 0; + +static void *dio_thread_entrance(void* arg); + +int storage_dio_init() +{ + int result; + int bytes; + int threads_count_per_path; + int context_count; + struct storage_dio_thread_data *pThreadData; + struct storage_dio_thread_data *pDataEnd; + struct storage_dio_context *pContext; + struct storage_dio_context *pContextEnd; + pthread_t tid; + pthread_attr_t thread_attr; + + if ((result=init_pthread_lock(&g_dio_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_attr(&thread_attr, g_thread_stack_size)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_attr fail, program exit!", __LINE__); + return result; + } + + bytes = sizeof(struct storage_dio_thread_data) * g_fdfs_store_paths.count; + g_dio_thread_data = (struct storage_dio_thread_data *)malloc(bytes); + if (g_dio_thread_data == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(g_dio_thread_data, 0, bytes); + + threads_count_per_path = g_disk_reader_threads + g_disk_writer_threads; + context_count = threads_count_per_path * g_fdfs_store_paths.count; + bytes = sizeof(struct storage_dio_context) * context_count; + g_dio_contexts = (struct storage_dio_context *)malloc(bytes); + if (g_dio_contexts == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(g_dio_contexts, 0, bytes); + + g_dio_thread_count = 0; + pDataEnd = g_dio_thread_data + g_fdfs_store_paths.count; + for (pThreadData=g_dio_thread_data; pThreadDatacount = threads_count_per_path; + pThreadData->contexts = g_dio_contexts + (pThreadData - \ + g_dio_thread_data) * threads_count_per_path; + pThreadData->reader = pThreadData->contexts; + pThreadData->writer = pThreadData->contexts+g_disk_reader_threads; + + pContextEnd = pThreadData->contexts + pThreadData->count; + for (pContext=pThreadData->contexts; pContextqueue))) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&(pContext->lock))) != 0) + { + return result; + } + + result = pthread_cond_init(&(pContext->cond), NULL); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "pthread_cond_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if ((result=pthread_create(&tid, &thread_attr, \ + dio_thread_entrance, pContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "create thread failed, " \ + "startup threads: %d, " \ + "errno: %d, error info: %s", \ + __LINE__, g_dio_thread_count, \ + result, STRERROR(result)); + return result; + } + else + { + pthread_mutex_lock(&g_dio_thread_lock); + g_dio_thread_count++; + pthread_mutex_unlock(&g_dio_thread_lock); + } + } + } + + pthread_attr_destroy(&thread_attr); + + return result; +} + +void storage_dio_terminate() +{ + struct storage_dio_context *pContext; + struct storage_dio_context *pContextEnd; + + pContextEnd = g_dio_contexts + g_dio_thread_count; + for (pContext=g_dio_contexts; pContextcond)); + } +} + +int storage_dio_queue_push(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + struct storage_dio_context *pContext; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + pContext = g_dio_contexts + pFileContext->dio_thread_index; + + pClientInfo->stage |= FDFS_STORAGE_STAGE_DIO_THREAD; + if ((result=task_queue_push(&(pContext->queue), pTask)) != 0) + { + add_to_deleted_list(pTask); + return result; + } + + if ((result=pthread_cond_signal(&(pContext->cond))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "pthread_cond_signal fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + + add_to_deleted_list(pTask); + return result; + } + + return 0; +} + +int storage_dio_get_thread_index(struct fast_task_info *pTask, \ + const int store_path_index, const char file_op) +{ + StorageClientInfo *pClientInfo; + struct storage_dio_thread_data *pThreadData; + struct storage_dio_context *contexts; + struct storage_dio_context *pContext; + int count; + + pClientInfo = (StorageClientInfo *)pTask->arg; + + pThreadData = g_dio_thread_data + store_path_index; + if (g_disk_rw_separated) + { + if (file_op == FDFS_STORAGE_FILE_OP_READ) + { + contexts = pThreadData->reader; + count = g_disk_reader_threads; + } + else + { + contexts = pThreadData->writer; + count = g_disk_writer_threads; + } + } + else + { + contexts = pThreadData->contexts; + count = pThreadData->count; + } + + pContext = contexts + (((unsigned int)pTask->event.fd) % count); + return pContext - g_dio_contexts; +} + +int dio_delete_normal_file(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + int result; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (unlink(pFileContext->filename) != 0) + { + result = errno != 0 ? errno : EACCES; + pFileContext->log_callback(pTask, result); + } + else + { + result = 0; + } + + pFileContext->done_callback(pTask, result); + return result; +} + +int dio_delete_trunk_file(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + int result; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + + if ((result=trunk_file_delete(pFileContext->filename, \ + &(pFileContext->extra_info.upload.trunk_info))) != 0) + { + pFileContext->log_callback(pTask, result); + } + + pFileContext->done_callback(pTask, result); + return result; +} + +int dio_discard_file(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + pFileContext->offset+=pTask->length - pFileContext->buff_offset; + if (pFileContext->offset >= pFileContext->end) + { + pFileContext->done_callback(pTask, 0); + } + else + { + pFileContext->buff_offset = 0; + storage_nio_notify(pTask); //notify nio to deal + } + + return 0; +} + +int dio_open_file(StorageFileContext *pFileContext) +{ + int result; + + if (pFileContext->fd >= 0) + { + return 0; + } + + pFileContext->fd = open(pFileContext->filename, + pFileContext->open_flags, 0644); + if (pFileContext->fd < 0) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "open file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + result, STRERROR(result)); + } + else + { + result = 0; + } + + pthread_mutex_lock(&g_dio_thread_lock); + g_storage_stat.total_file_open_count++; + if (result == 0) + { + g_storage_stat.success_file_open_count++; + } + pthread_mutex_unlock(&g_dio_thread_lock); + + if (result != 0) + { + return result; + } + + if (pFileContext->offset > 0 && lseek(pFileContext->fd, \ + pFileContext->offset, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "lseek file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + result, STRERROR(result)); + return result; + } + + return 0; +} + +int dio_read_file(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + int result; + int64_t remain_bytes; + int capacity_bytes; + int read_bytes; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + + do + { + if ((result=dio_open_file(pFileContext)) != 0) + { + break; + } + + remain_bytes = pFileContext->end - pFileContext->offset; + capacity_bytes = pTask->size - pTask->length; + read_bytes = (capacity_bytes < remain_bytes) ? \ + capacity_bytes : remain_bytes; + + /* + logInfo("###before dio read bytes: %d, pTask->length=%d, file offset=%ld", \ + read_bytes, pTask->length, pFileContext->offset); + */ + + if (read(pFileContext->fd, pTask->data + pTask->length, \ + read_bytes) != read_bytes) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "read from file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + result, STRERROR(result)); + } + + pthread_mutex_lock(&g_dio_thread_lock); + g_storage_stat.total_file_read_count++; + if (result == 0) + { + g_storage_stat.success_file_read_count++; + } + pthread_mutex_unlock(&g_dio_thread_lock); + + if (result != 0) + { + break; + } + + pTask->length += read_bytes; + pFileContext->offset += read_bytes; + + /* + logInfo("###after dio read bytes: %d, pTask->length=%d, file offset=%ld", \ + read_bytes, pTask->length, pFileContext->offset); + */ + + if (pFileContext->offset < pFileContext->end) + { + storage_nio_notify(pTask); //notify nio to deal + } + else + { + /* file read done, close it */ + close(pFileContext->fd); + pFileContext->fd = -1; + + pFileContext->done_callback(pTask, result); + } + + return 0; + + } while (0); + + /* file read error, close it */ + if (pFileContext->fd > 0) + { + close(pFileContext->fd); + pFileContext->fd = -1; + } + + pFileContext->done_callback(pTask, result); + return result; +} + +int dio_write_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int result; + int write_bytes; + char *pDataBuff; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + result = 0; + do + { + if (pFileContext->fd < 0) + { + if (pFileContext->extra_info.upload.before_open_callback!=NULL) + { + result = pFileContext->extra_info.upload. \ + before_open_callback(pTask); + if (result != 0) + { + break; + } + } + + if ((result=dio_open_file(pFileContext)) != 0) + { + break; + } + } + + pDataBuff = pTask->data + pFileContext->buff_offset; + write_bytes = pTask->length - pFileContext->buff_offset; + if (write(pFileContext->fd, pDataBuff, write_bytes) != write_bytes) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write to file: %s fail, fd=%d, write_bytes=%d, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + pFileContext->fd, write_bytes, \ + result, STRERROR(result)); + } + + pthread_mutex_lock(&g_dio_thread_lock); + g_storage_stat.total_file_write_count++; + if (result == 0) + { + g_storage_stat.success_file_write_count++; + } + pthread_mutex_unlock(&g_dio_thread_lock); + + if (result != 0) + { + break; + } + + if (pFileContext->calc_crc32) + { + pFileContext->crc32 = CRC32_ex(pDataBuff, write_bytes, \ + pFileContext->crc32); + } + + if (pFileContext->calc_file_hash) + { + if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH) + { + CALC_HASH_CODES4(pDataBuff, write_bytes, \ + pFileContext->file_hash_codes) + } + else + { + my_md5_update(&pFileContext->md5_context, \ + (unsigned char *)pDataBuff, write_bytes); + } + } + + /* + logInfo("###dio write bytes: %d, pTask->length=%d, buff_offset=%d", \ + write_bytes, pTask->length, pFileContext->buff_offset); + */ + + pFileContext->offset += write_bytes; + if (pFileContext->offset < pFileContext->end) + { + pFileContext->buff_offset = 0; + storage_nio_notify(pTask); //notify nio to deal + } + else + { + if (pFileContext->calc_crc32) + { + pFileContext->crc32 = CRC32_FINAL( \ + pFileContext->crc32); + } + + if (pFileContext->calc_file_hash) + { + if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH) + { + FINISH_HASH_CODES4(pFileContext->file_hash_codes) + } + else + { + my_md5_final((unsigned char *)(pFileContext-> \ + file_hash_codes), &pFileContext->md5_context); + } + } + + if (pFileContext->extra_info.upload.before_close_callback != NULL) + { + result = pFileContext->extra_info.upload. \ + before_close_callback(pTask); + } + + /* file write done, close it */ + close(pFileContext->fd); + pFileContext->fd = -1; + + if (pFileContext->done_callback != NULL) + { + pFileContext->done_callback(pTask, result); + } + } + + return 0; + } while (0); + + pClientInfo->clean_func(pTask); + + if (pFileContext->done_callback != NULL) + { + pFileContext->done_callback(pTask, result); + } + return result; +} + +int dio_truncate_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + result = 0; + do + { + if (pFileContext->fd < 0) + { + if (pFileContext->extra_info.upload.before_open_callback!=NULL) + { + result = pFileContext->extra_info.upload. \ + before_open_callback(pTask); + if (result != 0) + { + break; + } + } + + if ((result=dio_open_file(pFileContext)) != 0) + { + break; + } + } + + if (ftruncate(pFileContext->fd, pFileContext->offset) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "truncate file: %s fail, fd=%d, " \ + "remain_bytes="INT64_PRINTF_FORMAT", " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + pFileContext->fd, pFileContext->offset, \ + result, STRERROR(result)); + break; + } + + if (pFileContext->extra_info.upload.before_close_callback != NULL) + { + result = pFileContext->extra_info.upload. \ + before_close_callback(pTask); + } + + /* file write done, close it */ + close(pFileContext->fd); + pFileContext->fd = -1; + + if (pFileContext->done_callback != NULL) + { + pFileContext->done_callback(pTask, result); + } + + return 0; + } while (0); + + pClientInfo->clean_func(pTask); + + if (pFileContext->done_callback != NULL) + { + pFileContext->done_callback(pTask, result); + } + return result; +} + +void dio_read_finish_clean_up(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (pFileContext->fd > 0) + { + close(pFileContext->fd); + pFileContext->fd = -1; + } +} + +void dio_write_finish_clean_up(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (pFileContext->fd > 0) + { + close(pFileContext->fd); + pFileContext->fd = -1; + + /* if file does not write to the end, delete it */ + if (pFileContext->offset < pFileContext->end) + { + if (unlink(pFileContext->filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, " \ + "delete useless file %s fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + errno, STRERROR(errno)); + } + } + } +} + +void dio_append_finish_clean_up(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (pFileContext->fd > 0) + { + /* if file does not write to the end, + delete the appended contents + */ + if (pFileContext->offset > pFileContext->start && \ + pFileContext->offset < pFileContext->end) + { + if (ftruncate(pFileContext->fd,pFileContext->start)!=0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, " \ + "call ftruncate of file %s fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + errno, STRERROR(errno)); + } + else + { + logDebug("file: "__FILE__", line: %d, " \ + "client ip: %s, append file fail, " \ + "call ftruncate of file %s to size: "\ + INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + pFileContext->start); + } + } + + close(pFileContext->fd); + pFileContext->fd = -1; + } +} + +void dio_modify_finish_clean_up(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (pFileContext->fd > 0) + { + /* if file does not write to the end, log error info + */ + if (pFileContext->offset >= pFileContext->start && \ + pFileContext->offset < pFileContext->end) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, modify file: %s fail", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename); + } + + close(pFileContext->fd); + pFileContext->fd = -1; + } +} + +void dio_trunk_write_finish_clean_up(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + int result; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (pFileContext->fd > 0) + { + close(pFileContext->fd); + pFileContext->fd = -1; + + /* if file does not write to the end, + delete the appended contents + */ + if (pFileContext->offset > pFileContext->start && \ + pFileContext->offset < pFileContext->end) + { + if ((result=trunk_file_delete(pFileContext->filename, \ + &(pFileContext->extra_info.upload.trunk_info))) != 0) + { + } + } + } +} + +static void *dio_thread_entrance(void* arg) +{ + int result; + struct storage_dio_context *pContext; + struct fast_task_info *pTask; + + pContext = (struct storage_dio_context *)arg; + + pthread_mutex_lock(&(pContext->lock)); + while (g_continue_flag) + { + if ((result=pthread_cond_wait(&(pContext->cond), \ + &(pContext->lock))) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_cond_wait fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + while ((pTask=task_queue_pop(&(pContext->queue))) != NULL) + { + ((StorageClientInfo *)pTask->arg)->deal_func(pTask); + } + } + pthread_mutex_unlock(&(pContext->lock)); + + if ((result=pthread_mutex_lock(&g_dio_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + g_dio_thread_count--; + if ((result=pthread_mutex_unlock(&g_dio_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + logDebug("file: "__FILE__", line: %d, " \ + "dio thread exited, thread count: %d", \ + __LINE__, g_dio_thread_count); + + return NULL; +} + +int dio_check_trunk_file_when_upload(struct fast_task_info *pTask) +{ + int result; + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if ((result=trunk_check_and_init_file(pFileContext->filename)) != 0) + { + return result; + } + + if ((result=dio_open_file(pFileContext)) != 0) + { + return result; + } + + if (lseek(pFileContext->fd, -FDFS_TRUNK_FILE_HEADER_SIZE, SEEK_CUR) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "lseek file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + result, STRERROR(result)); + return result; + } + + return dio_check_trunk_file_ex(pFileContext->fd, pFileContext->filename, + pFileContext->start - FDFS_TRUNK_FILE_HEADER_SIZE); +} + +int dio_check_trunk_file_when_sync(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + return trunk_check_and_init_file(pFileContext->filename); +} + +int dio_check_trunk_file_ex(int fd, const char *filename, const int64_t offset) +{ + int result; + char old_header[FDFS_TRUNK_FILE_HEADER_SIZE]; + char expect_header[FDFS_TRUNK_FILE_HEADER_SIZE]; + + if (read(fd, old_header, FDFS_TRUNK_FILE_HEADER_SIZE) != + FDFS_TRUNK_FILE_HEADER_SIZE) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "read trunk header of file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + return result; + } + + memset(expect_header, 0, sizeof(expect_header)); + if (memcmp(old_header, expect_header, \ + FDFS_TRUNK_FILE_HEADER_SIZE) != 0) + { + FDFSTrunkHeader srcOldTrunkHeader; + FDFSTrunkHeader newOldTrunkHeader; + + trunk_unpack_header(old_header, &srcOldTrunkHeader); + memcpy(&newOldTrunkHeader, &srcOldTrunkHeader, \ + sizeof(FDFSTrunkHeader)); + newOldTrunkHeader.alloc_size = 0; + newOldTrunkHeader.file_size = 0; + newOldTrunkHeader.file_type = 0; + trunk_pack_header(&newOldTrunkHeader, old_header); + if (memcmp(old_header, expect_header, \ + FDFS_TRUNK_FILE_HEADER_SIZE) != 0) + { + char buff[256]; + trunk_header_dump(&srcOldTrunkHeader, \ + buff, sizeof(buff)); + + logError("file: "__FILE__", line: %d, " \ + "trunk file: %s, offset: " \ + INT64_PRINTF_FORMAT" already occupied" \ + " by other file, trunk header info: %s"\ + , __LINE__, filename, offset, buff); + return EEXIST; + } + } + + return 0; +} + +int dio_write_chunk_header(struct fast_task_info *pTask) +{ + StorageFileContext *pFileContext; + char header[FDFS_TRUNK_FILE_HEADER_SIZE]; + FDFSTrunkHeader trunkHeader; + int result; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_LINK) + { + trunkHeader.file_type = FDFS_TRUNK_FILE_TYPE_LINK; + } + else + { + trunkHeader.file_type = FDFS_TRUNK_FILE_TYPE_REGULAR; + } + + trunkHeader.alloc_size = pFileContext->extra_info.upload.trunk_info.file.size; + trunkHeader.file_size = pFileContext->end - pFileContext->start; + trunkHeader.crc32 = pFileContext->crc32; + trunkHeader.mtime = pFileContext->extra_info.upload.start_time; + snprintf(trunkHeader.formatted_ext_name, \ + sizeof(trunkHeader.formatted_ext_name), "%s", \ + pFileContext->extra_info.upload.formatted_ext_name); + + if (lseek(pFileContext->fd, pFileContext->start - \ + FDFS_TRUNK_FILE_HEADER_SIZE, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "lseek file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + result, STRERROR(result)); + return result; + } + + trunk_pack_header(&trunkHeader, header); + /* + { + char buff1[256]; + char buff2[256]; + char buff3[1024]; + trunk_header_dump(&trunkHeader, buff3, sizeof(buff3)); + logInfo("file: "__FILE__", line: %d, my trunk=%s, my fields=%s", __LINE__, \ + trunk_info_dump(&pFileContext->extra_info.upload.trunk_info, buff1, sizeof(buff1)), \ + trunk_header_dump(&trunkHeader, buff2, sizeof(buff2))); + } + */ + + if (write(pFileContext->fd, header, FDFS_TRUNK_FILE_HEADER_SIZE) != \ + FDFS_TRUNK_FILE_HEADER_SIZE) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write to file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pFileContext->filename, \ + result, STRERROR(result)); + return result; + } + + return 0; +} + diff --git a/storage/storage_dio.h b/storage/storage_dio.h new file mode 100644 index 0000000..74135c7 --- /dev/null +++ b/storage/storage_dio.h @@ -0,0 +1,77 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_dio.h + +#ifndef _STORAGE_DIO_H +#define _STORAGE_DIO_H + +#include +#include +#include +#include +#include "tracker_types.h" +#include "fast_task_queue.h" + +struct storage_dio_context +{ + struct fast_task_queue queue; + pthread_mutex_t lock; + pthread_cond_t cond; +}; + +struct storage_dio_thread_data +{ + /* for mixed read / write */ + struct storage_dio_context *contexts; + int count; //context count + + /* for separated read / write */ + struct storage_dio_context *reader; + struct storage_dio_context *writer; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_dio_thread_count; + +int storage_dio_init(); +void storage_dio_terminate(); + +int storage_dio_get_thread_index(struct fast_task_info *pTask, \ + const int store_path_index, const char file_op); +int storage_dio_queue_push(struct fast_task_info *pTask); + +int dio_read_file(struct fast_task_info *pTask); +int dio_write_file(struct fast_task_info *pTask); +int dio_truncate_file(struct fast_task_info *pTask); +int dio_delete_normal_file(struct fast_task_info *pTask); +int dio_delete_trunk_file(struct fast_task_info *pTask); +int dio_discard_file(struct fast_task_info *pTask); + +void dio_read_finish_clean_up(struct fast_task_info *pTask); +void dio_write_finish_clean_up(struct fast_task_info *pTask); +void dio_append_finish_clean_up(struct fast_task_info *pTask); +void dio_trunk_write_finish_clean_up(struct fast_task_info *pTask); +void dio_modify_finish_clean_up(struct fast_task_info *pTask); + +#define dio_truncate_finish_clean_up dio_read_finish_clean_up + +int dio_check_trunk_file_ex(int fd, const char *filename, const int64_t offset); +int dio_check_trunk_file_when_upload(struct fast_task_info *pTask); +int dio_check_trunk_file_when_sync(struct fast_task_info *pTask); +int dio_write_chunk_header(struct fast_task_info *pTask); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/storage_disk_recovery.c b/storage/storage_disk_recovery.c new file mode 100644 index 0000000..1ced1da --- /dev/null +++ b/storage/storage_disk_recovery.c @@ -0,0 +1,1111 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "avl_tree.h" +#include "shared_func.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_sync.h" +#include "tracker_client.h" +#include "storage_disk_recovery.h" +#include "storage_client.h" + +typedef struct { + char line[128]; + FDFSTrunkPathInfo path; //trunk file path + int id; //trunk file id +} FDFSTrunkFileIdInfo; + +#define RECOVERY_BINLOG_FILENAME ".binlog.recovery" +#define RECOVERY_MARK_FILENAME ".recovery.mark" + +#define MARK_ITEM_BINLOG_OFFSET "binlog_offset" +#define MARK_ITEM_FETCH_BINLOG_DONE "fetch_binlog_done" +#define MARK_ITEM_SAVED_STORAGE_STATUS "saved_storage_status" + +static int saved_storage_status = FDFS_STORAGE_STATUS_NONE; + +static char *recovery_get_binlog_filename(const void *pArg, \ + char *full_filename); + +static int storage_do_fetch_binlog(ConnectionInfo *pSrcStorage, \ + const int store_path_index) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + 1]; + char full_binlog_filename[MAX_PATH_SIZE]; + TrackerHeader *pHeader; + char *pBasePath; + int64_t in_bytes; + int64_t file_bytes; + int result; + + pBasePath = g_fdfs_store_paths.paths[store_path_index]; + recovery_get_binlog_filename(pBasePath, full_binlog_filename); + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + + long2buff(FDFS_GROUP_NAME_MAX_LEN + 1, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_FETCH_ONE_PATH_BINLOG; + strcpy(out_buff + sizeof(TrackerHeader), g_group_name); + *(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN) = \ + store_path_index; + + if((result=tcpsenddata_nb(pSrcStorage->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pSrcStorage->ip_addr, pSrcStorage->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=fdfs_recv_header(pSrcStorage, &in_bytes)) != 0) + { + return result; + } + + if ((result=tcprecvfile(pSrcStorage->sock, full_binlog_filename, \ + in_bytes, 0, g_fdfs_network_timeout, \ + &file_bytes)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, tcprecvfile fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pSrcStorage->ip_addr, pSrcStorage->port, \ + result, STRERROR(result)); + return result; + } + + logInfo("file: "__FILE__", line: %d, " \ + "recovery binlog file size: "INT64_PRINTF_FORMAT, \ + __LINE__, file_bytes); + + return 0; +} + +static int recovery_get_src_storage_server(ConnectionInfo *pSrcStorage) +{ + int result; + int storage_count; + ConnectionInfo trackerServer; + ConnectionInfo *pTrackerConn; + FDFSGroupStat groupStat; + FDFSStorageInfo storageStats[FDFS_MAX_SERVERS_EACH_GROUP]; + FDFSStorageInfo *pStorageStat; + FDFSStorageInfo *pStorageEnd; + + memset(pSrcStorage, 0, sizeof(ConnectionInfo)); + pSrcStorage->sock = -1; + + logDebug("file: "__FILE__", line: %d, " \ + "disk recovery: get source storage server", \ + __LINE__); + while (g_continue_flag) + { + result = tracker_get_storage_max_status(&g_tracker_group, \ + g_group_name, g_tracker_client_ip, \ + g_my_server_id_str, &saved_storage_status); + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "current storage: %s does not exist " \ + "in tracker server", __LINE__, \ + g_tracker_client_ip); + return ENOENT; + } + + if (result == 0) + { + if (saved_storage_status == FDFS_STORAGE_STATUS_INIT) + { + logInfo("file: "__FILE__", line: %d, " \ + "current storage: %s 's status is %d" \ + ", does not need recovery", __LINE__, \ + g_tracker_client_ip, \ + saved_storage_status); + return ENOENT; + } + + if (saved_storage_status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + saved_storage_status == FDFS_STORAGE_STATUS_DELETED) + { + logWarning("file: "__FILE__", line: %d, " \ + "current storage: %s 's status is %d" \ + ", does not need recovery", __LINE__, \ + g_tracker_client_ip, saved_storage_status); + return ENOENT; + } + + break; + } + + sleep(1); + } + + while (g_continue_flag) + { + if ((pTrackerConn=tracker_get_connection_r(&trackerServer, \ + &result)) == NULL) + { + sleep(5); + continue; + } + + result = tracker_list_one_group(pTrackerConn, \ + g_group_name, &groupStat); + if (result != 0) + { + tracker_disconnect_server_ex(pTrackerConn, true); + sleep(1); + continue; + } + + if (groupStat.count <= 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "storage server count: %d in the group <= 0!",\ + __LINE__, groupStat.count); + + tracker_disconnect_server(pTrackerConn); + sleep(1); + continue; + } + + if (groupStat.count == 1) + { + logInfo("file: "__FILE__", line: %d, " \ + "storage server count in the group = 1, " \ + "does not need recovery", __LINE__); + + tracker_disconnect_server(pTrackerConn); + return ENOENT; + } + + if (g_fdfs_store_paths.count > groupStat.store_path_count) + { + logInfo("file: "__FILE__", line: %d, " \ + "storage store path count: %d > " \ + "which of the group: %d, " \ + "does not need recovery", __LINE__, \ + g_fdfs_store_paths.count, groupStat.store_path_count); + + tracker_disconnect_server(pTrackerConn); + return ENOENT; + } + + if (groupStat.active_count <= 0) + { + tracker_disconnect_server(pTrackerConn); + sleep(5); + continue; + } + + result = tracker_list_servers(pTrackerConn, \ + g_group_name, NULL, storageStats, \ + FDFS_MAX_SERVERS_EACH_GROUP, &storage_count); + tracker_disconnect_server_ex(pTrackerConn, result != 0); + if (result != 0) + { + sleep(5); + continue; + } + + if (storage_count <= 1) + { + logWarning("file: "__FILE__", line: %d, " \ + "storage server count: %d in the group <= 1!",\ + __LINE__, storage_count); + + sleep(5); + continue; + } + + pStorageEnd = storageStats + storage_count; + for (pStorageStat=storageStats; pStorageStatid, g_my_server_id_str) == 0) + { + continue; + } + + if (pStorageStat->status == FDFS_STORAGE_STATUS_ACTIVE) + { + strcpy(pSrcStorage->ip_addr, \ + pStorageStat->ip_addr); + pSrcStorage->port = pStorageStat->storage_port; + break; + } + } + + if (pStorageStat < pStorageEnd) //found src storage server + { + break; + } + + sleep(5); + } + + if (!g_continue_flag) + { + return EINTR; + } + + logDebug("file: "__FILE__", line: %d, " \ + "disk recovery: get source storage server %s:%d", \ + __LINE__, pSrcStorage->ip_addr, pSrcStorage->port); + return 0; +} + +static char *recovery_get_full_filename(const void *pArg, \ + const char *filename, char *full_filename) +{ + const char *pBasePath; + static char buff[MAX_PATH_SIZE]; + + pBasePath = (const char *)pArg; + if (full_filename == NULL) + { + full_filename = buff; + } + + snprintf(full_filename, MAX_PATH_SIZE, \ + "%s/data/%s", pBasePath, filename); + + return full_filename; +} + +static char *recovery_get_binlog_filename(const void *pArg, \ + char *full_filename) +{ + return recovery_get_full_filename(pArg, \ + RECOVERY_BINLOG_FILENAME, full_filename); +} + +static char *recovery_get_mark_filename(const void *pArg, \ + char *full_filename) +{ + return recovery_get_full_filename(pArg, \ + RECOVERY_MARK_FILENAME, full_filename); +} + +static int storage_disk_recovery_finish(const char *pBasePath) +{ + char full_filename[MAX_PATH_SIZE]; + + recovery_get_binlog_filename(pBasePath, full_filename); + if (fileExists(full_filename)) + { + if (unlink(full_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "delete recovery binlog file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + } + + recovery_get_mark_filename(pBasePath, full_filename); + if (fileExists(full_filename)) + { + if (unlink(full_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "delete recovery mark file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + } + + return 0; +} + +static int recovery_write_to_mark_file(const char *pBasePath, \ + StorageBinLogReader *pReader) +{ + char buff[128]; + int len; + + len = sprintf(buff, \ + "%s=%d\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s=1\n", \ + MARK_ITEM_SAVED_STORAGE_STATUS, saved_storage_status, \ + MARK_ITEM_BINLOG_OFFSET, pReader->binlog_offset, \ + MARK_ITEM_FETCH_BINLOG_DONE); + + return storage_write_to_fd(pReader->mark_fd, \ + recovery_get_mark_filename, pBasePath, buff, len); +} + +static int recovery_init_binlog_file(const char *pBasePath) +{ + char full_binlog_filename[MAX_PATH_SIZE]; + char buff[1]; + + *buff = '\0'; + recovery_get_binlog_filename(pBasePath, full_binlog_filename); + return writeToFile(full_binlog_filename, buff, 0); +} + +static int recovery_init_mark_file(const char *pBasePath, \ + const bool fetch_binlog_done) +{ + char full_filename[MAX_PATH_SIZE]; + char buff[128]; + int len; + + recovery_get_mark_filename(pBasePath, full_filename); + + len = sprintf(buff, \ + "%s=%d\n" \ + "%s=0\n" \ + "%s=%d\n", \ + MARK_ITEM_SAVED_STORAGE_STATUS, saved_storage_status, \ + MARK_ITEM_BINLOG_OFFSET, \ + MARK_ITEM_FETCH_BINLOG_DONE, fetch_binlog_done); + return writeToFile(full_filename, buff, len); +} + +static int recovery_reader_init(const char *pBasePath, \ + StorageBinLogReader *pReader) +{ + char full_mark_filename[MAX_PATH_SIZE]; + IniContext iniContext; + int result; + + memset(pReader, 0, sizeof(StorageBinLogReader)); + pReader->mark_fd = -1; + pReader->binlog_fd = -1; + pReader->binlog_index = g_binlog_index + 1; + + pReader->binlog_buff.buffer = (char *)malloc( \ + STORAGE_BINLOG_BUFFER_SIZE); + if (pReader->binlog_buff.buffer == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, STORAGE_BINLOG_BUFFER_SIZE, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + pReader->binlog_buff.current = pReader->binlog_buff.buffer; + + recovery_get_mark_filename(pBasePath, full_mark_filename); + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(full_mark_filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load from mark file \"%s\" fail, " \ + "error code: %d", __LINE__, \ + full_mark_filename, result); + return result; + } + + if (!iniGetBoolValue(NULL, MARK_ITEM_FETCH_BINLOG_DONE, \ + &iniContext, false)) + { + iniFreeContext(&iniContext); + + logInfo("file: "__FILE__", line: %d, " \ + "mark file \"%s\", %s=0, " \ + "need to fetch binlog again", __LINE__, \ + full_mark_filename, MARK_ITEM_FETCH_BINLOG_DONE); + return EAGAIN; + } + + saved_storage_status = iniGetIntValue(NULL, \ + MARK_ITEM_SAVED_STORAGE_STATUS, &iniContext, -1); + if (saved_storage_status < 0) + { + iniFreeContext(&iniContext); + + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", %s: %d < 0", __LINE__, \ + full_mark_filename, MARK_ITEM_SAVED_STORAGE_STATUS, \ + saved_storage_status); + return EINVAL; + } + + pReader->binlog_offset = iniGetInt64Value(NULL, \ + MARK_ITEM_BINLOG_OFFSET, &iniContext, -1); + if (pReader->binlog_offset < 0) + { + iniFreeContext(&iniContext); + + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", %s: "\ + INT64_PRINTF_FORMAT" < 0", __LINE__, \ + full_mark_filename, MARK_ITEM_BINLOG_OFFSET, \ + pReader->binlog_offset); + return EINVAL; + } + + iniFreeContext(&iniContext); + + pReader->mark_fd = open(full_mark_filename, O_WRONLY | O_CREAT, 0644); + if (pReader->mark_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open mark file \"%s\" fail, " \ + "error no: %d, error info: %s", \ + __LINE__, full_mark_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if ((result=storage_open_readable_binlog(pReader, \ + recovery_get_binlog_filename, pBasePath)) != 0) + { + return result; + } + + return 0; +} + +static int storage_do_recovery(const char *pBasePath, StorageBinLogReader *pReader, \ + ConnectionInfo *pSrcStorage) +{ + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageConn; + FDFSTrunkFullInfo trunk_info; + StorageBinLogRecord record; + int record_length; + int result; + int log_level; + int count; + int store_path_index; + int64_t file_size; + int64_t total_count; + int64_t success_count; + bool bContinueFlag; + char local_filename[MAX_PATH_SIZE]; + char src_filename[MAX_PATH_SIZE]; + + pTrackerServer = g_tracker_group.servers; + count = 0; + total_count = 0; + success_count = 0; + result = 0; + + logInfo("file: "__FILE__", line: %d, " \ + "disk recovery: recovering files of data path: %s ...", \ + __LINE__, pBasePath); + + bContinueFlag = true; + while (bContinueFlag) + { + if ((pStorageConn=tracker_connect_server(pSrcStorage, &result)) == NULL) + { + sleep(5); + continue; + } + + while (g_continue_flag) + { + result=storage_binlog_read(pReader, &record, &record_length); + if (result != 0) + { + if (result == ENOENT) + { + result = 0; + } + bContinueFlag = false; + break; + } + + total_count++; + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_FILE) + { + bool bTrunkFile; + + if (fdfs_is_trunk_file(record.filename, \ + record.filename_len)) + { + char *pTrunkPathEnd; + char *pLocalFilename; + + bTrunkFile = true; + if (fdfs_decode_trunk_info(record.store_path_index, \ + record.true_filename, record.true_filename_len,\ + &trunk_info) != 0) + { + pReader->binlog_offset += record_length; + count++; + continue; + } + + trunk_get_full_filename(&trunk_info, \ + local_filename, sizeof(local_filename)); + + pTrunkPathEnd = strrchr(record.filename, '/'); + pLocalFilename = strrchr(local_filename, '/'); + if (pTrunkPathEnd == NULL || pLocalFilename == NULL) + { + pReader->binlog_offset += record_length; + count++; + continue; + } + sprintf(pTrunkPathEnd + 1, "%s", pLocalFilename + 1); + } + else + { + bTrunkFile = false; + sprintf(local_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[record.store_path_index], \ + record.true_filename); + } + + result = storage_download_file_to_file(pTrackerServer, \ + pStorageConn, g_group_name, \ + record.filename, local_filename, \ + &file_size); + if (result == 0) + { + if (!bTrunkFile) + { + set_file_utimes(local_filename, \ + record.timestamp); + } + + success_count++; + } + else if (result != ENOENT) + { + break; + } + } + else if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_LINK) + { + if (record.src_filename_len == 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid binlog line, filename: %s, " \ + "expect src filename", __LINE__, \ + record.filename); + result = EINVAL; + bContinueFlag = false; + break; + } + + if ((result=storage_split_filename_ex(record.filename, \ + &record.filename_len, record.true_filename, \ + &store_path_index)) != 0) + { + bContinueFlag = false; + break; + } + sprintf(local_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + record.true_filename); + + if ((result=storage_split_filename_ex( \ + record.src_filename, &record.src_filename_len,\ + record.true_filename, &store_path_index)) != 0) + { + bContinueFlag = false; + break; + } + sprintf(src_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + record.true_filename); + if (symlink(src_filename, local_filename) == 0) + { + success_count++; + } + else + { + result = errno != 0 ? errno : ENOENT; + if (result == ENOENT || result == EEXIST) + { + log_level = LOG_DEBUG; + } + else + { + log_level = LOG_ERR; + } + + log_it_ex(&g_log_context, log_level, \ + "file: "__FILE__", line: %d, " \ + "link file %s to %s fail, " \ + "errno: %d, error info: %s", __LINE__,\ + src_filename, local_filename, \ + result, STRERROR(result)); + + if (result != ENOENT && result != EEXIST) + { + bContinueFlag = false; + break; + } + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "invalid file op type: %d", \ + __LINE__, record.op_type); + result = EINVAL; + bContinueFlag = false; + break; + } + + pReader->binlog_offset += record_length; + count++; + if (count == 1000) + { + logDebug("file: "__FILE__", line: %d, " \ + "disk recovery: recover path: %s, " \ + "file count: "INT64_PRINTF_FORMAT \ + ", success count: "INT64_PRINTF_FORMAT, \ + __LINE__, pBasePath, total_count, \ + success_count); + recovery_write_to_mark_file(pBasePath, pReader); + count = 0; + } + } + + tracker_disconnect_server_ex(pStorageConn, result != 0); + if (count > 0) + { + recovery_write_to_mark_file(pBasePath, pReader); + count = 0; + + logInfo("file: "__FILE__", line: %d, " \ + "disk recovery: recover path: %s, " \ + "file count: "INT64_PRINTF_FORMAT \ + ", success count: "INT64_PRINTF_FORMAT, \ + __LINE__, pBasePath, total_count, success_count); + } + else + { + sleep(5); + } + } + + if (result == 0) + { + logInfo("file: "__FILE__", line: %d, " \ + "disk recovery: recover files of data path: %s done", \ + __LINE__, pBasePath); + } + + return result; +} + +int storage_disk_recovery_restore(const char *pBasePath) +{ + char full_binlog_filename[MAX_PATH_SIZE]; + char full_mark_filename[MAX_PATH_SIZE]; + ConnectionInfo srcStorage; + int result; + StorageBinLogReader reader; + + recovery_get_binlog_filename(pBasePath, full_binlog_filename); + recovery_get_mark_filename(pBasePath, full_mark_filename); + + if (!(fileExists(full_mark_filename) && \ + fileExists(full_binlog_filename))) + { + return 0; + } + + logInfo("file: "__FILE__", line: %d, " \ + "disk recovery: begin recovery data path: %s ...", \ + __LINE__, pBasePath); + + if ((result=recovery_get_src_storage_server(&srcStorage)) != 0) + { + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "no source storage server, " \ + "disk recovery finished!", __LINE__); + return storage_disk_recovery_finish(pBasePath); + } + else + { + return result; + } + } + + if ((result=recovery_reader_init(pBasePath, &reader)) != 0) + { + storage_reader_destroy(&reader); + return result; + } + + result = storage_do_recovery(pBasePath, &reader, &srcStorage); + + recovery_write_to_mark_file(pBasePath, &reader); + storage_reader_destroy(&reader); + + if (result != 0) + { + return result; + } + + while (g_continue_flag) + { + if (storage_report_storage_status(g_my_server_id_str, \ + g_tracker_client_ip, saved_storage_status) == 0) + { + break; + } + + sleep(5); + } + + if (!g_continue_flag) + { + return EINTR; + } + + logInfo("file: "__FILE__", line: %d, " \ + "disk recovery: end of recovery data path: %s", \ + __LINE__, pBasePath); + + return storage_disk_recovery_finish(pBasePath); +} + +static int storage_compare_trunk_id_info(void *p1, void *p2) +{ + int result; + result = memcmp(&(((FDFSTrunkFileIdInfo *)p1)->path), \ + &(((FDFSTrunkFileIdInfo *)p2)->path), \ + sizeof(FDFSTrunkPathInfo)); + if (result != 0) + { + return result; + } + + return ((FDFSTrunkFileIdInfo *)p1)->id - ((FDFSTrunkFileIdInfo *)p2)->id; +} + +static int tree_write_file_walk_callback(void *data, void *args) +{ + int result; + + if (fprintf((FILE *)args, "%s\n", \ + ((FDFSTrunkFileIdInfo *)data)->line) > 0) + { + return 0; + } + else + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write to binlog file fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, result, STRERROR(result)); + return EIO; + } +} + +static int storage_do_split_trunk_binlog(const int store_path_index, + StorageBinLogReader *pReader) +{ + FILE *fp; + char *pBasePath; + FDFSTrunkFileIdInfo *pFound; + char binlogFullFilename[MAX_PATH_SIZE]; + char tmpFullFilename[MAX_PATH_SIZE]; + FDFSTrunkFullInfo trunk_info; + FDFSTrunkFileIdInfo trunkFileId; + StorageBinLogRecord record; + AVLTreeInfo tree_unique_trunks; + int record_length; + int result; + + pBasePath = g_fdfs_store_paths.paths[store_path_index]; + recovery_get_full_filename(pBasePath, \ + RECOVERY_BINLOG_FILENAME".tmp", tmpFullFilename); + fp = fopen(tmpFullFilename, "w"); + if (fp == NULL) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "open file: %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, tmpFullFilename, + result, STRERROR(result)); + return result; + } + + if ((result=avl_tree_init(&tree_unique_trunks, free, \ + storage_compare_trunk_id_info)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "avl_tree_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + fclose(fp); + return result; + } + + memset(&trunk_info, 0, sizeof(trunk_info)); + memset(&trunkFileId, 0, sizeof(trunkFileId)); + result = 0; + while (g_continue_flag) + { + result=storage_binlog_read(pReader, &record, &record_length); + if (result != 0) + { + if (result == ENOENT) + { + result = 0; + } + break; + } + + if (fdfs_is_trunk_file(record.filename, record.filename_len)) + { + if (fdfs_decode_trunk_info(store_path_index, \ + record.true_filename, record.true_filename_len,\ + &trunk_info) != 0) + { + continue; + } + + trunkFileId.path = trunk_info.path; + trunkFileId.id = trunk_info.file.id; + pFound = (FDFSTrunkFileIdInfo *)avl_tree_find( \ + &tree_unique_trunks, &trunkFileId); + if (pFound != NULL) + { + continue; + } + + pFound = (FDFSTrunkFileIdInfo *)malloc( \ + sizeof(FDFSTrunkFileIdInfo)); + if (pFound == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__,\ + (int)sizeof(FDFSTrunkFileIdInfo), \ + result, STRERROR(result)); + break; + } + + sprintf(trunkFileId.line, "%d %c %s", \ + (int)record.timestamp, \ + record.op_type, record.filename); + memcpy(pFound, &trunkFileId, sizeof(FDFSTrunkFileIdInfo)); + if (avl_tree_insert(&tree_unique_trunks, pFound) != 1) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "avl_tree_insert fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + break; + } + } + else + { + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_FILE) + { + if (fprintf(fp, "%d %c %s\n", \ + (int)record.timestamp, \ + record.op_type, record.filename) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write to file: %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, tmpFullFilename, + result, STRERROR(result)); + break; + } + } + else + { + if (fprintf(fp, "%d %c %s %s\n", \ + (int)record.timestamp, \ + record.op_type, record.filename, \ + record.src_filename) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write to file: %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, tmpFullFilename, + result, STRERROR(result)); + break; + } + } + } + } + + if (result == 0) + { + int tree_node_count; + tree_node_count = avl_tree_count(&tree_unique_trunks); + if (tree_node_count > 0) + { + logInfo("file: "__FILE__", line: %d, " \ + "recovering trunk file count: %d", __LINE__, \ + tree_node_count); + + result = avl_tree_walk(&tree_unique_trunks, \ + tree_write_file_walk_callback, fp); + } + } + + avl_tree_destroy(&tree_unique_trunks); + fclose(fp); + if (!g_continue_flag) + { + return EINTR; + } + + if (result != 0) + { + return result; + } + + recovery_get_full_filename(pBasePath, \ + RECOVERY_BINLOG_FILENAME, binlogFullFilename); + if (rename(tmpFullFilename, binlogFullFilename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file %s to %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + tmpFullFilename, binlogFullFilename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + return 0; +} + +static int storage_disk_recovery_split_trunk_binlog(const int store_path_index) +{ + char *pBasePath; + StorageBinLogReader reader; + int result; + + pBasePath = g_fdfs_store_paths.paths[store_path_index]; + if ((result=recovery_reader_init(pBasePath, &reader)) != 0) + { + storage_reader_destroy(&reader); + return result; + } + + result = storage_do_split_trunk_binlog(store_path_index, &reader); + + storage_reader_destroy(&reader); + return result; +} + +int storage_disk_recovery_start(const int store_path_index) +{ + ConnectionInfo srcStorage; + ConnectionInfo *pStorageConn; + int result; + char *pBasePath; + + pBasePath = g_fdfs_store_paths.paths[store_path_index]; + if ((result=recovery_init_mark_file(pBasePath, false)) != 0) + { + return result; + } + + if ((result=recovery_init_binlog_file(pBasePath)) != 0) + { + return result; + } + + if ((result=recovery_get_src_storage_server(&srcStorage)) != 0) + { + if (result == ENOENT) + { + return storage_disk_recovery_finish(pBasePath); + } + else + { + return result; + } + } + + while (g_continue_flag) + { + if (storage_report_storage_status(g_my_server_id_str, \ + g_tracker_client_ip, FDFS_STORAGE_STATUS_RECOVERY) == 0) + { + break; + } + } + + if (!g_continue_flag) + { + return EINTR; + } + + if ((pStorageConn=tracker_connect_server(&srcStorage, &result)) == NULL) + { + return result; + } + + result = storage_do_fetch_binlog(pStorageConn, store_path_index); + tracker_disconnect_server_ex(pStorageConn, result != 0); + if (result != 0) + { + return result; + } + + //set fetch binlog done + if ((result=recovery_init_mark_file(pBasePath, true)) != 0) + { + return result; + } + + if ((result=storage_disk_recovery_split_trunk_binlog( \ + store_path_index)) != 0) + { + char markFullFilename[MAX_PATH_SIZE]; + unlink(recovery_get_mark_filename(pBasePath, markFullFilename)); + return result; + } + + return 0; +} + diff --git a/storage/storage_disk_recovery.h b/storage/storage_disk_recovery.h new file mode 100644 index 0000000..053f1dd --- /dev/null +++ b/storage/storage_disk_recovery.h @@ -0,0 +1,29 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_disk_recovery.h + +#ifndef _STORAGE_DISK_RECOVERY_H_ +#define _STORAGE_DISK_RECOVERY_H_ + +#include "tracker_types.h" +#include "tracker_client_thread.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int storage_disk_recovery_start(const int store_path_index); +int storage_disk_recovery_restore(const char *pBasePath); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/storage_dump.c b/storage/storage_dump.c new file mode 100644 index 0000000..adde16e --- /dev/null +++ b/storage/storage_dump.c @@ -0,0 +1,441 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include "storage_dump.h" +#include "shared_func.h" +#include "sched_thread.h" +#include "logger.h" +#include "hash.h" +#include "connection_pool.h" +#include "fdfs_global.h" +#include "storage_global.h" +#include "storage_service.h" +#include "storage_sync.h" +#include "trunk_mem.h" +#include "trunk_sync.h" + +static int fdfs_dump_global_vars(char *buff, const int buffSize) +{ + char szStorageJoinTime[32]; + char szSyncUntilTimestamp[32]; + char szUptime[32]; + char reserved_space_str[32]; + int total_len; + int i; + + total_len = snprintf(buff, buffSize, + "g_fdfs_connect_timeout=%ds\n" + "g_fdfs_network_timeout=%ds\n" + "g_fdfs_base_path=%s\n" + "g_fdfs_version=%d.%02d\n" + "g_continue_flag=%d\n" + "g_schedule_flag=%d\n" + "g_server_port=%d\n" + "g_max_connections=%d\n" + "g_storage_thread_count=%d\n" + "g_group_name=%s\n" + "g_sync_log_buff_interval=%d\n" + "g_subdir_count_per_path=%d\n" + "g_http_port=%d\n" + "g_last_server_port=%d\n" + "g_last_http_port=%d\n" + "g_allow_ip_count=%d\n" + "g_run_by_group=%s\n" + "g_run_by_user=%s\n" + "g_http_domain=%s\n" + "g_file_distribute_path_mode=%d\n" + "g_file_distribute_rotate_count=%d\n" + "g_fsync_after_written_bytes=%d\n" + "g_dist_path_index_high=%d\n" + "g_dist_path_index_low=%d\n" + "g_dist_write_file_count=%d\n" + "g_disk_rw_direct=%d\n" + "g_disk_rw_separated=%d\n" + "g_disk_reader_threads=%d\n" + "g_disk_writer_threads=%d\n" + "g_extra_open_file_flags=%d\n" + "g_tracker_reporter_count=%d\n" + "g_heart_beat_interval=%d\n" + "g_stat_report_interval=%d\n" + "g_sync_wait_usec=%dms\n" + "g_sync_interval=%dms\n" + "g_sync_start_time=%d:%d\n" + "g_sync_end_time=%d:%d\n" + "g_sync_part_time=%d\n" + "g_sync_log_buff_interval=%ds\n" + "g_sync_binlog_buff_interval=%ds\n" + "g_write_mark_file_freq=%d\n" + "g_sync_stat_file_interval=%ds\n" + "g_storage_join_time=%s\n" + "g_sync_old_done=%d\n" + "g_sync_src_id=%s\n" + "g_sync_until_timestamp=%s\n" + "g_my_server_id_str=%s\n" + "g_tracker_client_ip=%s\n" + "g_last_storage_ip=%s\n" + "g_check_file_duplicate=%d\n" + "g_key_namespace=%s\n" + "g_namespace_len=%d\n" + "g_bind_addr=%s\n" + "g_client_bind_addr=%d\n" + "g_storage_ip_changed_auto_adjust=%d\n" + "g_thread_kill_done=%d\n" + "g_thread_stack_size=%d\n" + "g_upload_priority=%d\n" + "g_up_time=%s\n" + "g_if_alias_prefix=%s\n" + "g_binlog_fd=%d\n" + "g_binlog_index=%d\n" + "g_storage_sync_thread_count=%d\n" + "g_use_storage_id=%d\n" + "g_if_use_trunk_file=%d\n" + "g_if_trunker_self=%d\n" + "g_slot_min_size=%d\n" + "g_trunk_file_size=%d\n" + "g_store_path_mode=%d\n" + "storage_reserved_mb=%s\n" + "g_avg_storage_reserved_mb=%d\n" + "g_store_path_index=%d\n" + "g_current_trunk_file_id=%d\n" + "g_trunk_sync_thread_count=%d\n" + "g_trunk_server=%s:%d\n" + "g_trunk_total_free_space="INT64_PRINTF_FORMAT"\n" + "g_use_connection_pool=%d\n" + "g_connection_pool_max_idle_time=%d\n" + "connection_pool_conn_count=%d\n" + #ifdef WITH_HTTPD + "g_http_params.disabled=%d\n" + "g_http_params.anti_steal_token=%d\n" + "g_http_params.server_port=%d\n" + "g_http_params.content_type_hash item count=%d\n" + "g_http_params.anti_steal_secret_key length=%d\n" + "g_http_params.token_check_fail_buff length=%d\n" + "g_http_params.default_content_type=%s\n" + "g_http_params.token_check_fail_content_type=%s\n" + "g_http_params.token_ttl=%d\n" + "g_http_trunk_size=%d\n" + #endif + #if defined(DEBUG_FLAG) && defined(OS_LINUX) + "g_exe_name=%s\n" + #endif + , g_fdfs_connect_timeout + , g_fdfs_network_timeout + , g_fdfs_base_path + , g_fdfs_version.major, g_fdfs_version.minor + , g_continue_flag + , g_schedule_flag + , g_server_port + , g_max_connections + , g_storage_thread_count + , g_group_name + , g_sync_log_buff_interval + , g_subdir_count_per_path + , g_http_port + , g_last_server_port + , g_last_http_port + , g_allow_ip_count + , g_run_by_group + , g_run_by_user + , g_http_domain + , g_file_distribute_path_mode + , g_file_distribute_rotate_count + , g_fsync_after_written_bytes + , g_dist_path_index_high + , g_dist_path_index_low + , g_dist_write_file_count + , g_disk_rw_direct + , g_disk_rw_separated + , g_disk_reader_threads + , g_disk_writer_threads + , g_extra_open_file_flags + , g_tracker_reporter_count + , g_heart_beat_interval + , g_stat_report_interval + , g_sync_wait_usec / 1000 + , g_sync_interval + , g_sync_start_time.hour, g_sync_start_time.minute + , g_sync_end_time.hour, g_sync_end_time.minute + , g_sync_part_time + , g_sync_log_buff_interval + , g_sync_binlog_buff_interval + , g_write_mark_file_freq + , g_sync_stat_file_interval + , formatDatetime(g_storage_join_time, "%Y-%m-%d %H:%M:%S", + szStorageJoinTime, sizeof(szStorageJoinTime)) + , g_sync_old_done + , g_sync_src_id + , formatDatetime(g_sync_until_timestamp, "%Y-%m-%d %H:%M:%S", + szSyncUntilTimestamp, sizeof(szSyncUntilTimestamp)) + , g_my_server_id_str + , g_tracker_client_ip + , g_last_storage_ip + , g_check_file_duplicate + , g_key_namespace + , g_namespace_len + , g_bind_addr + , g_client_bind_addr + , g_storage_ip_changed_auto_adjust + , g_thread_kill_done + , g_thread_stack_size + , g_upload_priority + , formatDatetime(g_up_time, "%Y-%m-%d %H:%M:%S", + szUptime, sizeof(szUptime)) + , g_if_alias_prefix + , g_binlog_fd + , g_binlog_index + , g_storage_sync_thread_count + , g_use_storage_id + , g_if_use_trunk_file + , g_if_trunker_self + , g_slot_min_size + , g_trunk_file_size + , g_store_path_mode + , fdfs_storage_reserved_space_to_string( \ + &g_storage_reserved_space, reserved_space_str) \ + , g_avg_storage_reserved_mb + , g_store_path_index + , g_current_trunk_file_id + , g_trunk_sync_thread_count + , g_trunk_server.ip_addr, g_trunk_server.port + , g_trunk_total_free_space + , g_use_connection_pool + , g_connection_pool_max_idle_time + , g_use_connection_pool ? conn_pool_get_connection_count( \ + &g_connection_pool) : 0 + #ifdef WITH_HTTPD + , g_http_params.disabled + , g_http_params.anti_steal_token + , g_http_params.server_port + , hash_count(&(g_http_params.content_type_hash)) + , g_http_params.anti_steal_secret_key.length + , g_http_params.token_check_fail_buff.length + , g_http_params.default_content_type + , g_http_params.token_check_fail_content_type + , g_http_params.token_ttl + , g_http_trunk_size + #endif + + #if defined(DEBUG_FLAG) && defined(OS_LINUX) + , g_exe_name + #endif + ); + + total_len += snprintf(buff + total_len, buffSize - total_len, + "\ng_fdfs_store_paths.count=%d\n", g_fdfs_store_paths.count); + for (i=0; iip_addr, pTrackerServer->port); + } + + return total_len; +} + +static int fdfs_dump_storage_servers(char *buff, const int buffSize) +{ + int total_len; + char szLastSyncSrcTimestamp[32]; + FDFSStorageServer *pServer; + FDFSStorageServer *pServerEnd; + FDFSStorageServer **ppServer; + FDFSStorageServer **ppServerEnd; + + total_len = snprintf(buff, buffSize, + "\ng_storage_count=%d\n", g_storage_count); + pServerEnd = g_storage_servers + g_storage_count; + for (pServer=g_storage_servers; pServerserver.ip_addr, pServer->server.status, + formatDatetime(pServer->last_sync_src_timestamp, + "%Y-%m-%d %H:%M:%S", szLastSyncSrcTimestamp, + sizeof(szLastSyncSrcTimestamp))); + } + + total_len += snprintf(buff + total_len, buffSize - total_len, + "sorted storage servers:\n"); + ppServerEnd = g_sorted_storages + g_storage_count; + for (ppServer=g_sorted_storages; ppServerserver.ip_addr); + } + + return total_len; +} + +static int fdfs_dump_storage_stat(char *buff, const int buffSize) +{ + int total_len; + char szLastHeartBeatTime[32]; + char szSrcUpdTime[32]; + char szSyncUpdTime[32]; + char szSyncedTimestamp[32]; + + total_len = snprintf(buff, buffSize, + "\ng_stat_change_count=%d\n" + "g_sync_change_count=%d\n" + "total_upload_count="INT64_PRINTF_FORMAT"\n" + "success_upload_count="INT64_PRINTF_FORMAT"\n" + "total_set_meta_count="INT64_PRINTF_FORMAT"\n" + "success_set_meta_count="INT64_PRINTF_FORMAT"\n" + "total_delete_count="INT64_PRINTF_FORMAT"\n" + "success_delete_count="INT64_PRINTF_FORMAT"\n" + "total_download_count="INT64_PRINTF_FORMAT"\n" + "success_download_count="INT64_PRINTF_FORMAT"\n" + "total_get_meta_count="INT64_PRINTF_FORMAT"\n" + "success_get_meta_count="INT64_PRINTF_FORMAT"\n" + "total_create_link_count="INT64_PRINTF_FORMAT"\n" + "success_create_link_count="INT64_PRINTF_FORMAT"\n" + "total_delete_link_count="INT64_PRINTF_FORMAT"\n" + "success_delete_link_count="INT64_PRINTF_FORMAT"\n" + "last_source_update=%s\n" + "last_sync_update=%s\n" + "last_synced_timestamp=%s\n" + "last_heart_beat_time=%s\n", + g_stat_change_count, g_sync_change_count, + g_storage_stat.total_upload_count, + g_storage_stat.success_upload_count, + g_storage_stat.total_set_meta_count, + g_storage_stat.success_set_meta_count, + g_storage_stat.total_delete_count, + g_storage_stat.success_delete_count, + g_storage_stat.total_download_count, + g_storage_stat.success_download_count, + g_storage_stat.total_get_meta_count, + g_storage_stat.success_get_meta_count, + g_storage_stat.total_create_link_count, + g_storage_stat.success_create_link_count, + g_storage_stat.total_delete_link_count, + g_storage_stat.success_delete_link_count, + formatDatetime(g_storage_stat.last_source_update, + "%Y-%m-%d %H:%M:%S", + szSrcUpdTime, sizeof(szSrcUpdTime)), + formatDatetime(g_storage_stat.last_sync_update, + "%Y-%m-%d %H:%M:%S", + szSyncUpdTime, sizeof(szSyncUpdTime)), + formatDatetime(g_storage_stat.last_synced_timestamp, + "%Y-%m-%d %H:%M:%S", + szSyncedTimestamp, sizeof(szSyncedTimestamp)), + formatDatetime(g_storage_stat.last_heart_beat_time, + "%Y-%m-%d %H:%M:%S", + szLastHeartBeatTime, sizeof(szLastHeartBeatTime))); + + return total_len; +} + +#define WRITE_TO_FILE(fd, buff, len) \ + if (write(fd, buff, len) != len) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "write to file %s fail, errno: %d, error info: %s", \ + __LINE__, filename, errno, STRERROR(errno)); \ + result = errno; \ + break; \ + } + +int fdfs_dump_storage_global_vars_to_file(const char *filename) +{ + char buff[4 * 1024]; + char szCurrentTime[32]; + int len; + int result; + int fd; + + fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd < 0) + { + logError("file: "__FILE__", line: %d, " + "open file %s fail, errno: %d, error info: %s", + __LINE__, filename, errno, STRERROR(errno)); + return errno; + } + + do + { + result = 0; + formatDatetime(g_current_time, "%Y-%m-%d %H:%M:%S", + szCurrentTime, sizeof(szCurrentTime)); + + len = sprintf(buff, "\n====time: %s DUMP START====\n", + szCurrentTime); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_global_vars(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_tracker_servers(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_storage_stat(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_storage_servers(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = sprintf(buff, "\n====time: %s DUMP END====\n\n", + szCurrentTime); + WRITE_TO_FILE(fd, buff, len) + } while(0); + + close(fd); + + return result; +} + diff --git a/storage/storage_dump.h b/storage/storage_dump.h new file mode 100644 index 0000000..4da03a0 --- /dev/null +++ b/storage/storage_dump.h @@ -0,0 +1,30 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_dump.h + +#ifndef _STORAGE_DUMP_H +#define _STORAGE_DUMP_H + +#include +#include +#include +#include "fdfs_define.h" +#include "tracker_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int fdfs_dump_storage_global_vars_to_file(const char *filename); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/storage_func.c b/storage/storage_func.c new file mode 100644 index 0000000..753025d --- /dev/null +++ b/storage/storage_func.c @@ -0,0 +1,2128 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_func.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "ini_file_reader.h" +#include "connection_pool.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "fdfs_shared_func.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_param_getter.h" +#include "storage_ip_changed_dealer.h" +#include "fdht_global.h" +#include "fdht_func.h" +#include "fdht_client.h" +#include "client_func.h" +#include "trunk_mem.h" +#include "trunk_sync.h" +#include "storage_disk_recovery.h" +#include "tracker_client.h" + +#ifdef WITH_HTTPD +#include "fdfs_http_shared.h" +#endif + +#define DATA_DIR_INITED_FILENAME ".data_init_flag" +#define STORAGE_STAT_FILENAME "storage_stat.dat" + +#define INIT_ITEM_STORAGE_JOIN_TIME "storage_join_time" +#define INIT_ITEM_SYNC_OLD_DONE "sync_old_done" +#define INIT_ITEM_SYNC_SRC_SERVER "sync_src_server" +#define INIT_ITEM_SYNC_UNTIL_TIMESTAMP "sync_until_timestamp" +#define INIT_ITEM_LAST_IP_ADDRESS "last_ip_addr" +#define INIT_ITEM_LAST_SERVER_PORT "last_server_port" +#define INIT_ITEM_LAST_HTTP_PORT "last_http_port" +#define INIT_ITEM_CURRENT_TRUNK_FILE_ID "current_trunk_file_id" +#define INIT_ITEM_TRUNK_LAST_COMPRESS_TIME "trunk_last_compress_time" + +#define STAT_ITEM_TOTAL_UPLOAD "total_upload_count" +#define STAT_ITEM_SUCCESS_UPLOAD "success_upload_count" +#define STAT_ITEM_TOTAL_APPEND "total_append_count" +#define STAT_ITEM_SUCCESS_APPEND "success_append_count" +#define STAT_ITEM_TOTAL_MODIFY "total_modify_count" +#define STAT_ITEM_SUCCESS_MODIFY "success_modify_count" +#define STAT_ITEM_TOTAL_TRUNCATE "total_truncate_count" +#define STAT_ITEM_SUCCESS_TRUNCATE "success_truncate_count" +#define STAT_ITEM_TOTAL_DOWNLOAD "total_download_count" +#define STAT_ITEM_SUCCESS_DOWNLOAD "success_download_count" +#define STAT_ITEM_LAST_SOURCE_UPD "last_source_update" +#define STAT_ITEM_LAST_SYNC_UPD "last_sync_update" +#define STAT_ITEM_TOTAL_SET_META "total_set_meta_count" +#define STAT_ITEM_SUCCESS_SET_META "success_set_meta_count" +#define STAT_ITEM_TOTAL_DELETE "total_delete_count" +#define STAT_ITEM_SUCCESS_DELETE "success_delete_count" +#define STAT_ITEM_TOTAL_GET_META "total_get_meta_count" +#define STAT_ITEM_SUCCESS_GET_META "success_get_meta_count" +#define STAT_ITEM_TOTAL_CREATE_LINK "total_create_link_count" +#define STAT_ITEM_SUCCESS_CREATE_LINK "success_create_link_count" +#define STAT_ITEM_TOTAL_DELETE_LINK "total_delete_link_count" +#define STAT_ITEM_SUCCESS_DELETE_LINK "success_delete_link_count" +#define STAT_ITEM_TOTAL_UPLOAD_BYTES "total_upload_bytes" +#define STAT_ITEM_SUCCESS_UPLOAD_BYTES "success_upload_bytes" +#define STAT_ITEM_TOTAL_APPEND_BYTES "total_append_bytes" +#define STAT_ITEM_SUCCESS_APPEND_BYTES "success_append_bytes" +#define STAT_ITEM_TOTAL_MODIFY_BYTES "total_modify_bytes" +#define STAT_ITEM_SUCCESS_MODIFY_BYTES "success_modify_bytes" +#define STAT_ITEM_TOTAL_DOWNLOAD_BYTES "total_download_bytes" +#define STAT_ITEM_SUCCESS_DOWNLOAD_BYTES "success_download_bytes" +#define STAT_ITEM_TOTAL_SYNC_IN_BYTES "total_sync_in_bytes" +#define STAT_ITEM_SUCCESS_SYNC_IN_BYTES "success_sync_in_bytes" +#define STAT_ITEM_TOTAL_SYNC_OUT_BYTES "total_sync_out_bytes" +#define STAT_ITEM_SUCCESS_SYNC_OUT_BYTES "success_sync_out_bytes" +#define STAT_ITEM_TOTAL_FILE_OPEN_COUNT "total_file_open_count" +#define STAT_ITEM_SUCCESS_FILE_OPEN_COUNT "success_file_open_count" +#define STAT_ITEM_TOTAL_FILE_READ_COUNT "total_file_read_count" +#define STAT_ITEM_SUCCESS_FILE_READ_COUNT "success_file_read_count" +#define STAT_ITEM_TOTAL_FILE_WRITE_COUNT "total_file_write_count" +#define STAT_ITEM_SUCCESS_FILE_WRITE_COUNT "success_file_write_count" + +#define STAT_ITEM_DIST_PATH_INDEX_HIGH "dist_path_index_high" +#define STAT_ITEM_DIST_PATH_INDEX_LOW "dist_path_index_low" +#define STAT_ITEM_DIST_WRITE_FILE_COUNT "dist_write_file_count" + +static int storage_stat_fd = -1; + +/* +static pthread_mutex_t fsync_thread_mutex; +static pthread_cond_t fsync_thread_cond; +static int fsync_thread_count = 0; +*/ + +static pthread_mutex_t sync_stat_file_lock; + +static int storage_open_stat_file(); +static int storage_close_stat_file(); +static int storage_make_data_dirs(const char *pBasePath, bool *pathCreated); +static int storage_check_and_make_data_dirs(); + +static int tracker_get_my_server_id() +{ + struct in_addr ip_addr; + + if (inet_pton(AF_INET, g_tracker_client_ip, &ip_addr) == 1) + { + g_server_id_in_filename = ip_addr.s_addr; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "call inet_pton for ip: %s fail", \ + __LINE__,g_tracker_client_ip); + g_server_id_in_filename = INADDR_NONE; + } + + if (g_use_storage_id) + { + ConnectionInfo *pTrackerServer; + int result; + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + return errno != 0 ? errno : ECONNREFUSED; + } + + result = tracker_get_storage_id(pTrackerServer, \ + g_group_name, g_tracker_client_ip, g_my_server_id_str); + tracker_disconnect_server_ex(pTrackerServer, result != 0); + if (result != 0) + { + return result; + } + + if (g_id_type_in_filename == FDFS_ID_TYPE_SERVER_ID) + { + g_server_id_in_filename = atoi(g_my_server_id_str); + } + } + else + { + snprintf(g_my_server_id_str, sizeof(g_my_server_id_str), "%s", \ + g_tracker_client_ip); + } + + logInfo("file: "__FILE__", line: %d, " \ + "tracker_client_ip: %s, my_server_id_str: %s, " \ + "g_server_id_in_filename: %d", __LINE__, \ + g_tracker_client_ip, g_my_server_id_str, g_server_id_in_filename); + return 0; +} + +static char *get_storage_stat_filename(const void *pArg, char *full_filename) +{ + static char buff[MAX_PATH_SIZE]; + + if (full_filename == NULL) + { + full_filename = buff; + } + + snprintf(full_filename, MAX_PATH_SIZE, \ + "%s/data/%s", g_fdfs_base_path, STORAGE_STAT_FILENAME); + return full_filename; +} + +int storage_write_to_fd(int fd, get_filename_func filename_func, \ + const void *pArg, const char *buff, const int len) +{ + if (ftruncate(fd, 0) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "truncate file \"%s\" to empty fail, " \ + "error no: %d, error info: %s", \ + __LINE__, filename_func(pArg, NULL), \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (lseek(fd, 0, SEEK_SET) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rewind file \"%s\" to start fail, " \ + "error no: %d, error info: %s", \ + __LINE__, filename_func(pArg, NULL), \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "error no: %d, error info: %s", \ + __LINE__, filename_func(pArg, NULL), \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (fsync(fd) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync file \"%s\" to disk fail, " \ + "error no: %d, error info: %s", \ + __LINE__, filename_func(pArg, NULL), \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + return 0; +} + +static int storage_open_stat_file() +{ + char full_filename[MAX_PATH_SIZE]; + IniContext iniContext; + int result; + + get_storage_stat_filename(NULL, full_filename); + if (fileExists(full_filename)) + { + if ((result=iniLoadFromFile(full_filename, &iniContext)) \ + != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load from stat file \"%s\" fail, " \ + "error code: %d", \ + __LINE__, full_filename, result); + return result; + } + + if (iniContext.global.count < 12) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in stat file \"%s\", item count: %d < 12", \ + __LINE__, full_filename, iniContext.global.count); + return ENOENT; + } + + g_storage_stat.total_upload_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_UPLOAD, &iniContext, 0); + g_storage_stat.success_upload_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_UPLOAD, &iniContext, 0); + g_storage_stat.total_append_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_APPEND, &iniContext, 0); + g_storage_stat.success_append_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_APPEND, &iniContext, 0); + g_storage_stat.total_modify_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_MODIFY, &iniContext, 0); + g_storage_stat.success_modify_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_MODIFY, &iniContext, 0); + g_storage_stat.total_truncate_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_TRUNCATE, &iniContext, 0); + g_storage_stat.success_truncate_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_TRUNCATE, &iniContext, 0); + g_storage_stat.total_download_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_DOWNLOAD, &iniContext, 0); + g_storage_stat.success_download_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_DOWNLOAD, &iniContext, 0); + g_storage_stat.last_source_update = iniGetIntValue(NULL, \ + STAT_ITEM_LAST_SOURCE_UPD, &iniContext, 0); + g_storage_stat.last_sync_update = iniGetIntValue(NULL, \ + STAT_ITEM_LAST_SYNC_UPD, &iniContext, 0); + g_storage_stat.total_set_meta_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_SET_META, &iniContext, 0); + g_storage_stat.success_set_meta_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_SET_META, &iniContext, 0); + g_storage_stat.total_delete_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_DELETE, &iniContext, 0); + g_storage_stat.success_delete_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_DELETE, &iniContext, 0); + g_storage_stat.total_get_meta_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_GET_META, &iniContext, 0); + g_storage_stat.success_get_meta_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_GET_META, &iniContext, 0); + g_storage_stat.total_create_link_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_CREATE_LINK, &iniContext, 0); + g_storage_stat.success_create_link_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_CREATE_LINK, &iniContext, 0); + g_storage_stat.total_delete_link_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_DELETE_LINK, &iniContext, 0); + g_storage_stat.success_delete_link_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_DELETE_LINK, &iniContext, 0); + g_storage_stat.total_upload_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_UPLOAD_BYTES, &iniContext, 0); + g_storage_stat.success_upload_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_UPLOAD_BYTES, &iniContext, 0); + g_storage_stat.total_append_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_APPEND_BYTES, &iniContext, 0); + g_storage_stat.success_append_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_APPEND_BYTES, &iniContext, 0); + g_storage_stat.total_modify_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_MODIFY_BYTES, &iniContext, 0); + g_storage_stat.success_modify_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_MODIFY_BYTES, &iniContext, 0); + g_storage_stat.total_download_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_DOWNLOAD_BYTES, &iniContext, 0); + g_storage_stat.success_download_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_DOWNLOAD_BYTES, &iniContext, 0); + g_storage_stat.total_sync_in_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_SYNC_IN_BYTES, &iniContext, 0); + g_storage_stat.success_sync_in_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_SYNC_IN_BYTES, &iniContext, 0); + g_storage_stat.total_sync_out_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_SYNC_OUT_BYTES, &iniContext, 0); + g_storage_stat.success_sync_out_bytes = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_SYNC_OUT_BYTES, &iniContext, 0); + g_storage_stat.total_file_open_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_FILE_OPEN_COUNT, &iniContext, 0); + g_storage_stat.success_file_open_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_FILE_OPEN_COUNT, &iniContext, 0); + g_storage_stat.total_file_read_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_FILE_READ_COUNT, &iniContext, 0); + g_storage_stat.success_file_read_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_FILE_READ_COUNT, &iniContext, 0); + g_storage_stat.total_file_write_count = iniGetInt64Value(NULL, \ + STAT_ITEM_TOTAL_FILE_WRITE_COUNT, &iniContext, 0); + g_storage_stat.success_file_write_count = iniGetInt64Value(NULL, \ + STAT_ITEM_SUCCESS_FILE_WRITE_COUNT, &iniContext, 0); + g_dist_path_index_high = iniGetIntValue(NULL, \ + STAT_ITEM_DIST_PATH_INDEX_HIGH, &iniContext, 0); + g_dist_path_index_low = iniGetIntValue(NULL, \ + STAT_ITEM_DIST_PATH_INDEX_LOW, &iniContext, 0); + g_dist_write_file_count = iniGetIntValue(NULL, \ + STAT_ITEM_DIST_WRITE_FILE_COUNT, &iniContext, 0); + + iniFreeContext(&iniContext); + } + else + { + memset(&g_storage_stat, 0, sizeof(g_storage_stat)); + } + + storage_stat_fd = open(full_filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (storage_stat_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open stat file \"%s\" fail, " \ + "error no: %d, error info: %s", \ + __LINE__, full_filename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if ((result=storage_write_to_stat_file()) != 0) + { + return result; + } + + STORAGE_FCHOWN(storage_stat_fd, full_filename, geteuid(), getegid()) + return 0; +} + +static int storage_close_stat_file() +{ + int result; + + result = 0; + if (storage_stat_fd >= 0) + { + result = storage_write_to_stat_file(); + if (close(storage_stat_fd) != 0) + { + result += errno != 0 ? errno : ENOENT; + } + storage_stat_fd = -1; + } + + return result; +} + +int storage_write_to_stat_file() +{ + char buff[2048]; + int len; + int result; + int write_ret; + + len = sprintf(buff, + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s=%d\n", \ + STAT_ITEM_TOTAL_UPLOAD, g_storage_stat.total_upload_count, \ + STAT_ITEM_SUCCESS_UPLOAD, g_storage_stat.success_upload_count, \ + STAT_ITEM_TOTAL_APPEND, g_storage_stat.total_append_count, \ + STAT_ITEM_SUCCESS_APPEND, g_storage_stat.success_append_count, \ + STAT_ITEM_TOTAL_MODIFY, g_storage_stat.total_modify_count, \ + STAT_ITEM_SUCCESS_MODIFY, g_storage_stat.success_modify_count, \ + STAT_ITEM_TOTAL_TRUNCATE, g_storage_stat.total_truncate_count, \ + STAT_ITEM_SUCCESS_TRUNCATE, g_storage_stat.success_truncate_count, \ + STAT_ITEM_TOTAL_DOWNLOAD, g_storage_stat.total_download_count, \ + STAT_ITEM_SUCCESS_DOWNLOAD, \ + g_storage_stat.success_download_count, \ + STAT_ITEM_TOTAL_SET_META, g_storage_stat.total_set_meta_count, \ + STAT_ITEM_SUCCESS_SET_META, \ + g_storage_stat.success_set_meta_count, \ + STAT_ITEM_TOTAL_DELETE, g_storage_stat.total_delete_count, \ + STAT_ITEM_SUCCESS_DELETE, g_storage_stat.success_delete_count, \ + STAT_ITEM_TOTAL_GET_META, g_storage_stat.total_get_meta_count, \ + STAT_ITEM_SUCCESS_GET_META, \ + g_storage_stat.success_get_meta_count, \ + STAT_ITEM_TOTAL_CREATE_LINK, \ + g_storage_stat.total_create_link_count, \ + STAT_ITEM_SUCCESS_CREATE_LINK, \ + g_storage_stat.success_create_link_count, \ + STAT_ITEM_TOTAL_DELETE_LINK, \ + g_storage_stat.total_delete_link_count, \ + STAT_ITEM_SUCCESS_DELETE_LINK, \ + g_storage_stat.success_delete_link_count, \ + STAT_ITEM_TOTAL_UPLOAD_BYTES, \ + g_storage_stat.total_upload_bytes, + STAT_ITEM_SUCCESS_UPLOAD_BYTES, \ + g_storage_stat.success_upload_bytes, \ + STAT_ITEM_TOTAL_APPEND_BYTES, \ + g_storage_stat.total_append_bytes, + STAT_ITEM_SUCCESS_APPEND_BYTES, \ + g_storage_stat.success_append_bytes, \ + STAT_ITEM_TOTAL_MODIFY_BYTES, \ + g_storage_stat.total_modify_bytes, + STAT_ITEM_SUCCESS_MODIFY_BYTES, \ + g_storage_stat.success_modify_bytes, \ + STAT_ITEM_TOTAL_DOWNLOAD_BYTES, \ + g_storage_stat.total_download_bytes, \ + STAT_ITEM_SUCCESS_DOWNLOAD_BYTES, \ + g_storage_stat.success_download_bytes, \ + STAT_ITEM_TOTAL_SYNC_IN_BYTES, \ + g_storage_stat.total_sync_in_bytes, \ + STAT_ITEM_SUCCESS_SYNC_IN_BYTES, \ + g_storage_stat.success_sync_in_bytes, \ + STAT_ITEM_TOTAL_SYNC_OUT_BYTES, \ + g_storage_stat.total_sync_out_bytes, \ + STAT_ITEM_SUCCESS_SYNC_OUT_BYTES, \ + g_storage_stat.success_sync_out_bytes, \ + STAT_ITEM_TOTAL_FILE_OPEN_COUNT, \ + g_storage_stat.total_file_open_count, \ + STAT_ITEM_SUCCESS_FILE_OPEN_COUNT, \ + g_storage_stat.success_file_open_count, \ + STAT_ITEM_TOTAL_FILE_READ_COUNT, \ + g_storage_stat.total_file_read_count, \ + STAT_ITEM_SUCCESS_FILE_READ_COUNT, \ + g_storage_stat.success_file_read_count, \ + STAT_ITEM_TOTAL_FILE_WRITE_COUNT, \ + g_storage_stat.total_file_write_count, \ + STAT_ITEM_SUCCESS_FILE_WRITE_COUNT, \ + g_storage_stat.success_file_write_count, \ + STAT_ITEM_LAST_SOURCE_UPD, \ + (int)g_storage_stat.last_source_update, \ + STAT_ITEM_LAST_SYNC_UPD, (int)g_storage_stat.last_sync_update,\ + STAT_ITEM_DIST_PATH_INDEX_HIGH, g_dist_path_index_high, \ + STAT_ITEM_DIST_PATH_INDEX_LOW, g_dist_path_index_low, \ + STAT_ITEM_DIST_WRITE_FILE_COUNT, g_dist_write_file_count + ); + + if ((result=pthread_mutex_lock(&sync_stat_file_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + write_ret = storage_write_to_fd(storage_stat_fd, \ + get_storage_stat_filename, NULL, buff, len); + + if ((result=pthread_mutex_unlock(&sync_stat_file_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return write_ret; +} + +int storage_write_to_sync_ini_file() +{ + char full_filename[MAX_PATH_SIZE]; + char buff[512]; + int fd; + int len; + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/%s", g_fdfs_base_path, DATA_DIR_INITED_FILENAME); + if ((fd=open(full_filename, O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + len = sprintf(buff, "%s=%d\n" \ + "%s=%d\n" \ + "%s=%s\n" \ + "%s=%d\n" \ + "%s=%s\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s=%d\n", \ + INIT_ITEM_STORAGE_JOIN_TIME, g_storage_join_time, \ + INIT_ITEM_SYNC_OLD_DONE, g_sync_old_done, \ + INIT_ITEM_SYNC_SRC_SERVER, g_sync_src_id, \ + INIT_ITEM_SYNC_UNTIL_TIMESTAMP, g_sync_until_timestamp, \ + INIT_ITEM_LAST_IP_ADDRESS, g_tracker_client_ip, \ + INIT_ITEM_LAST_SERVER_PORT, g_last_server_port, \ + INIT_ITEM_LAST_HTTP_PORT, g_last_http_port, + INIT_ITEM_CURRENT_TRUNK_FILE_ID, g_current_trunk_file_id, \ + INIT_ITEM_TRUNK_LAST_COMPRESS_TIME, (int)g_trunk_last_compress_time + ); + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + close(fd); + return errno != 0 ? errno : EIO; + } + + close(fd); + + STORAGE_CHOWN(full_filename, geteuid(), getegid()) + + return 0; +} + +static int storage_check_and_make_data_dirs() +{ + int result; + int i; + char data_path[MAX_PATH_SIZE]; + char full_filename[MAX_PATH_SIZE]; + bool pathCreated; + + snprintf(data_path, sizeof(data_path), "%s/data", \ + g_fdfs_base_path); + snprintf(full_filename, sizeof(full_filename), "%s/%s", \ + data_path, DATA_DIR_INITED_FILENAME); + if (fileExists(full_filename)) + { + IniContext iniContext; + char *pValue; + + if ((result=iniLoadFromFile(full_filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load from file \"%s/%s\" fail, " \ + "error code: %d", \ + __LINE__, data_path, \ + full_filename, result); + return result; + } + + pValue = iniGetStrValue(NULL, INIT_ITEM_STORAGE_JOIN_TIME, \ + &iniContext); + if (pValue == NULL) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in file \"%s/%s\", item \"%s\" not exists", \ + __LINE__, data_path, full_filename, \ + INIT_ITEM_STORAGE_JOIN_TIME); + return ENOENT; + } + g_storage_join_time = atoi(pValue); + + pValue = iniGetStrValue(NULL, INIT_ITEM_SYNC_OLD_DONE, \ + &iniContext); + if (pValue == NULL) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in file \"%s/%s\", item \"%s\" not exists", \ + __LINE__, data_path, full_filename, \ + INIT_ITEM_SYNC_OLD_DONE); + return ENOENT; + } + g_sync_old_done = atoi(pValue); + + pValue = iniGetStrValue(NULL, INIT_ITEM_SYNC_SRC_SERVER, \ + &iniContext); + if (pValue == NULL) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in file \"%s/%s\", item \"%s\" not exists", \ + __LINE__, data_path, full_filename, \ + INIT_ITEM_SYNC_SRC_SERVER); + return ENOENT; + } + snprintf(g_sync_src_id, sizeof(g_sync_src_id), \ + "%s", pValue); + + g_sync_until_timestamp = iniGetIntValue(NULL, \ + INIT_ITEM_SYNC_UNTIL_TIMESTAMP, \ + &iniContext, 0); + + pValue = iniGetStrValue(NULL, INIT_ITEM_LAST_IP_ADDRESS, \ + &iniContext); + if (pValue != NULL) + { + snprintf(g_last_storage_ip, sizeof(g_last_storage_ip), \ + "%s", pValue); + } + + pValue = iniGetStrValue(NULL, INIT_ITEM_LAST_SERVER_PORT, \ + &iniContext); + if (pValue != NULL) + { + g_last_server_port = atoi(pValue); + } + + pValue = iniGetStrValue(NULL, INIT_ITEM_LAST_HTTP_PORT, \ + &iniContext); + if (pValue != NULL) + { + g_last_http_port = atoi(pValue); + } + + g_current_trunk_file_id = iniGetIntValue(NULL, \ + INIT_ITEM_CURRENT_TRUNK_FILE_ID, &iniContext, 0); + g_trunk_last_compress_time = iniGetIntValue(NULL, \ + INIT_ITEM_TRUNK_LAST_COMPRESS_TIME , &iniContext, 0); + + iniFreeContext(&iniContext); + + if (g_last_server_port == 0 || g_last_http_port == 0) + { + if (g_last_server_port == 0) + { + g_last_server_port = g_server_port; + } + + if (g_last_http_port == 0) + { + g_last_http_port = g_http_port; + } + + if ((result=storage_write_to_sync_ini_file()) != 0) + { + return result; + } + } + + /* + logInfo("g_sync_old_done = %d, " + "g_sync_src_id = %s, " + "g_sync_until_timestamp = %d, " + "g_last_storage_ip = %s, " + "g_last_server_port = %d, " + "g_last_http_port = %d, " + "g_current_trunk_file_id = %d, " + "g_trunk_last_compress_time = %d", + g_sync_old_done, g_sync_src_id, g_sync_until_timestamp, + g_last_storage_ip, g_last_server_port, g_last_http_port, + g_current_trunk_file_id, (int)g_trunk_last_compress_time + ); + */ + } + else + { + if (!fileExists(data_path)) + { + if (mkdir(data_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + STORAGE_CHOWN(data_path, geteuid(), getegid()) + } + + g_last_server_port = g_server_port; + g_last_http_port = g_http_port; + g_storage_join_time = g_current_time; + if ((result=storage_write_to_sync_ini_file()) != 0) + { + return result; + } + } + + for (i=0; i 0) + { + sleep(1); + } + */ + + if ((result=iniLoadFromFile(filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, ret code: %d", \ + __LINE__, filename, result); + return result; + } + + do + { + if (iniGetBoolValue(NULL, "disabled", &iniContext, false)) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" disabled=true, exit", \ + __LINE__, filename); + result = ECANCELED; + break; + } + + g_subdir_count_per_path=iniGetIntValue(NULL, \ + "subdir_count_per_path", &iniContext, \ + DEFAULT_DATA_DIR_COUNT_PER_PATH); + if (g_subdir_count_per_path <= 0 || \ + g_subdir_count_per_path > 256) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", invalid subdir_count: %d", \ + __LINE__, filename, g_subdir_count_per_path); + result = EINVAL; + break; + } + + if ((result=storage_load_paths(&iniContext)) != 0) + { + break; + } + + load_log_level(&iniContext); + if ((result=log_set_prefix(g_fdfs_base_path, \ + STORAGE_ERROR_LOG_FILENAME)) != 0) + { + break; + } + + g_fdfs_connect_timeout = iniGetIntValue(NULL, "connect_timeout", \ + &iniContext, DEFAULT_CONNECT_TIMEOUT); + if (g_fdfs_connect_timeout <= 0) + { + g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT; + } + + g_fdfs_network_timeout = iniGetIntValue(NULL, "network_timeout", \ + &iniContext, DEFAULT_NETWORK_TIMEOUT); + if (g_fdfs_network_timeout <= 0) + { + g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT; + } + g_network_tv.tv_sec = g_fdfs_network_timeout; + + g_server_port = iniGetIntValue(NULL, "port", &iniContext, \ + FDFS_STORAGE_SERVER_DEF_PORT); + if (g_server_port <= 0) + { + g_server_port = FDFS_STORAGE_SERVER_DEF_PORT; + } + + g_heart_beat_interval = iniGetIntValue(NULL, \ + "heart_beat_interval", &iniContext, \ + STORAGE_BEAT_DEF_INTERVAL); + if (g_heart_beat_interval <= 0) + { + g_heart_beat_interval = STORAGE_BEAT_DEF_INTERVAL; + } + + g_stat_report_interval = iniGetIntValue(NULL, \ + "stat_report_interval", &iniContext, \ + STORAGE_REPORT_DEF_INTERVAL); + if (g_stat_report_interval <= 0) + { + g_stat_report_interval = STORAGE_REPORT_DEF_INTERVAL; + } + + pBindAddr = iniGetStrValue(NULL, "bind_addr", &iniContext); + if (pBindAddr == NULL) + { + *bind_addr = '\0'; + } + else + { + snprintf(bind_addr, addr_size, "%s", pBindAddr); + } + + g_client_bind_addr = iniGetBoolValue(NULL, "client_bind", \ + &iniContext, true); + + pGroupName = iniGetStrValue(NULL, "group_name", &iniContext); + if (pGroupName == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" must have item " \ + "\"group_name\"!", \ + __LINE__, filename); + result = ENOENT; + break; + } + if (pGroupName[0] == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "group_name is empty!", \ + __LINE__, filename); + result = EINVAL; + break; + } + + snprintf(g_group_name, sizeof(g_group_name), "%s", pGroupName); + if ((result=fdfs_validate_group_name(g_group_name)) != 0) \ + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "the group name \"%s\" is invalid!", \ + __LINE__, filename, g_group_name); + result = EINVAL; + break; + } + + result = fdfs_load_tracker_group_ex(&g_tracker_group, \ + filename, &iniContext); + if (result != 0) + { + break; + } + + pEnd = g_tracker_group.servers + g_tracker_group.server_count; + for (pServer=g_tracker_group.servers; pServerip_addr, pServer->port); + if (strcmp(pServer->ip_addr, "127.0.0.1") == 0) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "tracker: \"%s:%d\" is invalid, " \ + "tracker server ip can't be 127.0.0.1",\ + __LINE__, filename, pServer->ip_addr, \ + pServer->port); + result = EINVAL; + break; + } + } + if (result != 0) + { + break; + } + + g_sync_wait_usec = iniGetIntValue(NULL, "sync_wait_msec",\ + &iniContext, STORAGE_DEF_SYNC_WAIT_MSEC); + if (g_sync_wait_usec <= 0) + { + g_sync_wait_usec = STORAGE_DEF_SYNC_WAIT_MSEC; + } + g_sync_wait_usec *= 1000; + + g_sync_interval = iniGetIntValue(NULL, "sync_interval",\ + &iniContext, 0); + if (g_sync_interval < 0) + { + g_sync_interval = 0; + } + g_sync_interval *= 1000; + + if ((result=get_time_item_from_conf(&iniContext, \ + "sync_start_time", &g_sync_start_time, 0, 0)) != 0) + { + break; + } + if ((result=get_time_item_from_conf(&iniContext, \ + "sync_end_time", &g_sync_end_time, 23, 59)) != 0) + { + break; + } + + g_sync_part_time = !((g_sync_start_time.hour == 0 && \ + g_sync_start_time.minute == 0) && \ + (g_sync_end_time.hour == 23 && \ + g_sync_end_time.minute == 59)); + + g_max_connections = iniGetIntValue(NULL, "max_connections", \ + &iniContext, DEFAULT_MAX_CONNECTONS); + if (g_max_connections <= 0) + { + g_max_connections = DEFAULT_MAX_CONNECTONS; + } + if ((result=set_rlimit(RLIMIT_NOFILE, g_max_connections)) != 0) + { + break; + } + + g_accept_threads = iniGetIntValue(NULL, "accept_threads", \ + &iniContext, 1); + if (g_accept_threads <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"accept_threads\" is invalid, " \ + "value: %d <= 0!", __LINE__, g_accept_threads); + result = EINVAL; + break; + } + + g_work_threads = iniGetIntValue(NULL, "work_threads", \ + &iniContext, DEFAULT_WORK_THREADS); + if (g_work_threads <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"work_threads\" is invalid, " \ + "value: %d <= 0!", __LINE__, g_work_threads); + result = EINVAL; + break; + } + + pBuffSize = iniGetStrValue(NULL, \ + "buff_size", &iniContext); + if (pBuffSize == NULL) + { + buff_size = STORAGE_DEFAULT_BUFF_SIZE; + } + else if ((result=parse_bytes(pBuffSize, 1, &buff_size)) != 0) + { + break; + } + g_buff_size = buff_size; + if (g_buff_size < 4 * 1024 || \ + g_buff_size < sizeof(TrackerHeader) + \ + TRUNK_BINLOG_BUFFER_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "item \"buff_size\" is too small, " \ + "value: %d < %d or < %d!", __LINE__, \ + g_buff_size, 4 * 1024, \ + (int)sizeof(TrackerHeader) + \ + TRUNK_BINLOG_BUFFER_SIZE); + result = EINVAL; + break; + } + + g_disk_rw_separated = iniGetBoolValue(NULL, \ + "disk_rw_separated", &iniContext, true); + + g_disk_reader_threads = iniGetIntValue(NULL, \ + "disk_reader_threads", \ + &iniContext, DEFAULT_DISK_READER_THREADS); + if (g_disk_reader_threads < 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"disk_reader_threads\" is invalid, " \ + "value: %d < 0!", __LINE__, \ + g_disk_reader_threads); + result = EINVAL; + break; + } + + g_disk_writer_threads = iniGetIntValue(NULL, \ + "disk_writer_threads", \ + &iniContext, DEFAULT_DISK_WRITER_THREADS); + if (g_disk_writer_threads < 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"disk_writer_threads\" is invalid, " \ + "value: %d < 0!", __LINE__, \ + g_disk_writer_threads); + result = EINVAL; + break; + } + + if (g_disk_rw_separated) + { + if (g_disk_reader_threads == 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"disk_reader_threads\" is " \ + "invalid, value = 0!", __LINE__); + result = EINVAL; + break; + } + + if (g_disk_writer_threads == 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"disk_writer_threads\" is " \ + "invalid, value = 0!", __LINE__); + result = EINVAL; + break; + } + } + else if (g_disk_reader_threads + g_disk_writer_threads == 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"disk_reader_threads\" and " \ + "\"disk_writer_threads\" are " \ + "invalid, both value = 0!", __LINE__); + result = EINVAL; + break; + } + + /* + g_disk_rw_direct = iniGetBoolValue(NULL, \ + "disk_rw_direct", &iniContext, false); + */ + + pRunByGroup = iniGetStrValue(NULL, "run_by_group", &iniContext); + pRunByUser = iniGetStrValue(NULL, "run_by_user", &iniContext); + if (pRunByGroup == NULL) + { + *g_run_by_group = '\0'; + } + else + { + snprintf(g_run_by_group, sizeof(g_run_by_group), \ + "%s", pRunByGroup); + } + if (*g_run_by_group == '\0') + { + g_run_by_gid = getegid(); + } + else + { + struct group *pGroup; + + pGroup = getgrnam(g_run_by_group); + if (pGroup == NULL) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "getgrnam fail, errno: %d, " \ + "error info: %s", __LINE__, \ + result, STRERROR(result)); + return result; + } + + g_run_by_gid = pGroup->gr_gid; + } + + if (pRunByUser == NULL) + { + *g_run_by_user = '\0'; + } + else + { + snprintf(g_run_by_user, sizeof(g_run_by_user), \ + "%s", pRunByUser); + } + if (*g_run_by_user == '\0') + { + g_run_by_uid = geteuid(); + } + else + { + struct passwd *pUser; + + pUser = getpwnam(g_run_by_user); + if (pUser == NULL) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "getpwnam fail, errno: %d, " \ + "error info: %s", __LINE__, \ + result, STRERROR(result)); + return result; + } + + g_run_by_uid = pUser->pw_uid; + } + + if ((result=load_allow_hosts(&iniContext, \ + &g_allow_ip_addrs, &g_allow_ip_count)) != 0) + { + return result; + } + + g_file_distribute_path_mode = iniGetIntValue(NULL, \ + "file_distribute_path_mode", &iniContext, \ + FDFS_FILE_DIST_PATH_ROUND_ROBIN); + g_file_distribute_rotate_count = iniGetIntValue(NULL, \ + "file_distribute_rotate_count", &iniContext, \ + FDFS_FILE_DIST_DEFAULT_ROTATE_COUNT); + if (g_file_distribute_rotate_count <= 0) + { + g_file_distribute_rotate_count = \ + FDFS_FILE_DIST_DEFAULT_ROTATE_COUNT; + } + + pFsyncAfterWrittenBytes = iniGetStrValue(NULL, \ + "fsync_after_written_bytes", &iniContext); + if (pFsyncAfterWrittenBytes == NULL) + { + fsync_after_written_bytes = 0; + } + else if ((result=parse_bytes(pFsyncAfterWrittenBytes, 1, \ + &fsync_after_written_bytes)) != 0) + { + break; + } + g_fsync_after_written_bytes = fsync_after_written_bytes; + + g_sync_log_buff_interval = iniGetIntValue(NULL, \ + "sync_log_buff_interval", &iniContext, \ + SYNC_LOG_BUFF_DEF_INTERVAL); + if (g_sync_log_buff_interval <= 0) + { + g_sync_log_buff_interval = SYNC_LOG_BUFF_DEF_INTERVAL; + } + + g_sync_binlog_buff_interval = iniGetIntValue(NULL, \ + "sync_binlog_buff_interval", &iniContext,\ + SYNC_BINLOG_BUFF_DEF_INTERVAL); + if (g_sync_binlog_buff_interval <= 0) + { + g_sync_binlog_buff_interval=SYNC_BINLOG_BUFF_DEF_INTERVAL; + } + + g_write_mark_file_freq = iniGetIntValue(NULL, \ + "write_mark_file_freq", &iniContext, \ + FDFS_DEFAULT_SYNC_MARK_FILE_FREQ); + if (g_write_mark_file_freq <= 0) + { + g_write_mark_file_freq = FDFS_DEFAULT_SYNC_MARK_FILE_FREQ; + } + + + g_sync_stat_file_interval = iniGetIntValue(NULL, \ + "sync_stat_file_interval", &iniContext, \ + DEFAULT_SYNC_STAT_FILE_INTERVAL); + if (g_sync_stat_file_interval <= 0) + { + g_sync_stat_file_interval=DEFAULT_SYNC_STAT_FILE_INTERVAL; + } + + pThreadStackSize = iniGetStrValue(NULL, \ + "thread_stack_size", &iniContext); + if (pThreadStackSize == NULL) + { + thread_stack_size = 512 * 1024; + } + else if ((result=parse_bytes(pThreadStackSize, 1, \ + &thread_stack_size)) != 0) + { + break; + } + g_thread_stack_size = (int)thread_stack_size; + + if (g_thread_stack_size < 64 * 1024) + { + logError("file: "__FILE__", line: %d, " \ + "item \"thread_stack_size\" %d is invalid, " \ + "which < %d", __LINE__, g_thread_stack_size, \ + 64 * 1024); + result = EINVAL; + break; + } + + g_upload_priority = iniGetIntValue(NULL, \ + "upload_priority", &iniContext, \ + DEFAULT_UPLOAD_PRIORITY); + + pIfAliasPrefix = iniGetStrValue(NULL, \ + "if_alias_prefix", &iniContext); + if (pIfAliasPrefix == NULL) + { + *g_if_alias_prefix = '\0'; + } + else + { + snprintf(g_if_alias_prefix, sizeof(g_if_alias_prefix), + "%s", pIfAliasPrefix); + } + + g_check_file_duplicate = iniGetBoolValue(NULL, \ + "check_file_duplicate", &iniContext, false); + if (g_check_file_duplicate) + { + char *pKeyNamespace; + char *pFileSignatureMethod; + + pFileSignatureMethod = iniGetStrValue(NULL, \ + "file_signature_method", &iniContext); + if (pFileSignatureMethod != NULL && strcasecmp( \ + pFileSignatureMethod, "md5") == 0) + { + g_file_signature_method = \ + STORAGE_FILE_SIGNATURE_METHOD_MD5; + } + else + { + g_file_signature_method = \ + STORAGE_FILE_SIGNATURE_METHOD_HASH; + } + + strcpy(g_fdht_base_path, g_fdfs_base_path); + g_fdht_connect_timeout = g_fdfs_connect_timeout; + g_fdht_network_timeout = g_fdfs_network_timeout; + + pKeyNamespace = iniGetStrValue(NULL, \ + "key_namespace", &iniContext); + if (pKeyNamespace == NULL || *pKeyNamespace == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "item \"key_namespace\" does not " \ + "exist or is empty", __LINE__); + result = EINVAL; + break; + } + + g_namespace_len = strlen(pKeyNamespace); + if (g_namespace_len >= sizeof(g_key_namespace)) + { + g_namespace_len = sizeof(g_key_namespace) - 1; + } + memcpy(g_key_namespace, pKeyNamespace, g_namespace_len); + *(g_key_namespace + g_namespace_len) = '\0'; + + if ((result=fdht_load_groups(&iniContext, \ + &g_group_array)) != 0) + { + break; + } + + g_keep_alive = iniGetBoolValue(NULL, "keep_alive", \ + &iniContext, false); + } + + g_http_port = iniGetIntValue(NULL, "http.server_port", \ + &iniContext, 80); + if (g_http_port <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "invalid param \"http.server_port\": %d", \ + __LINE__, g_http_port); + result = EINVAL; + break; + } + + pHttpDomain = iniGetStrValue(NULL, \ + "http.domain_name", &iniContext); + if (pHttpDomain == NULL) + { + *g_http_domain = '\0'; + } + else + { + snprintf(g_http_domain, sizeof(g_http_domain), \ + "%s", pHttpDomain); + } + + g_use_access_log = iniGetBoolValue(NULL, "use_access_log", \ + &iniContext, false); + if (g_use_access_log) + { + result = log_init_ex(&g_access_log_context); + if (result != 0) + { + break; + } + + log_set_time_precision(&g_access_log_context, \ + LOG_TIME_PRECISION_MSECOND); + log_set_cache_ex(&g_access_log_context, true); + result = log_set_prefix_ex(&g_access_log_context, \ + g_fdfs_base_path, "storage_access"); + if (result != 0) + { + break; + } + } + + g_rotate_access_log = iniGetBoolValue(NULL, "rotate_access_log",\ + &iniContext, false); + if ((result=get_time_item_from_conf(&iniContext, \ + "access_log_rotate_time", &g_access_log_rotate_time, \ + 0, 0)) != 0) + { + break; + } + + g_rotate_error_log = iniGetBoolValue(NULL, "rotate_error_log",\ + &iniContext, false); + if ((result=get_time_item_from_conf(&iniContext, \ + "error_log_rotate_time", &g_error_log_rotate_time, \ + 0, 0)) != 0) + { + break; + } + + pRotateAccessLogSize = iniGetStrValue(NULL, \ + "rotate_access_log_size", &iniContext); + if (pRotateAccessLogSize == NULL) + { + rotate_access_log_size = 0; + } + else if ((result=parse_bytes(pRotateAccessLogSize, 1, \ + &rotate_access_log_size)) != 0) + { + break; + } + if (rotate_access_log_size > 0 && \ + rotate_access_log_size< FDFS_ONE_MB) + { + logWarning("file: "__FILE__", line: %d, " \ + "item \"rotate_access_log_size\": " \ + INT64_PRINTF_FORMAT" is too small, " \ + "change to 1 MB", __LINE__, \ + rotate_access_log_size); + rotate_access_log_size = FDFS_ONE_MB; + } + g_access_log_context.rotate_size = rotate_access_log_size; + + pRotateErrorLogSize = iniGetStrValue(NULL, \ + "rotate_error_log_size", &iniContext); + if (pRotateErrorLogSize == NULL) + { + rotate_error_log_size = 0; + } + else if ((result=parse_bytes(pRotateErrorLogSize, 1, \ + &rotate_error_log_size)) != 0) + { + break; + } + if (rotate_error_log_size > 0 && \ + rotate_error_log_size < FDFS_ONE_MB) + { + logWarning("file: "__FILE__", line: %d, " \ + "item \"rotate_error_log_size\": " \ + INT64_PRINTF_FORMAT" is too small, " \ + "change to 1 MB", __LINE__, \ + rotate_error_log_size); + rotate_error_log_size = FDFS_ONE_MB; + } + g_log_context.rotate_size = rotate_error_log_size; + + g_file_sync_skip_invalid_record = iniGetBoolValue(NULL, \ + "file_sync_skip_invalid_record", &iniContext, false); + + if ((result=fdfs_connection_pool_init(filename, &iniContext)) != 0) + { + break; + } + +#ifdef WITH_HTTPD + { + char *pHttpTrunkSize; + int64_t http_trunk_size; + + if ((result=fdfs_http_params_load(&iniContext, \ + filename, &g_http_params)) != 0) + { + break; + } + + pHttpTrunkSize = iniGetStrValue(NULL, \ + "http.trunk_size", &iniContext); + if (pHttpTrunkSize == NULL) + { + http_trunk_size = 64 * 1024; + } + else if ((result=parse_bytes(pHttpTrunkSize, 1, \ + &http_trunk_size)) != 0) + { + break; + } + + g_http_trunk_size = (int)http_trunk_size; + } +#endif + + logInfo("FastDFS v%d.%02d, base_path=%s, store_path_count=%d, " \ + "subdir_count_per_path=%d, group_name=%s, " \ + "run_by_group=%s, run_by_user=%s, " \ + "connect_timeout=%ds, network_timeout=%ds, "\ + "port=%d, bind_addr=%s, client_bind=%d, " \ + "max_connections=%d, accept_threads=%d, " \ + "work_threads=%d, " \ + "disk_rw_separated=%d, disk_reader_threads=%d, " \ + "disk_writer_threads=%d, " \ + "buff_size=%dKB, heart_beat_interval=%ds, " \ + "stat_report_interval=%ds, tracker_server_count=%d, " \ + "sync_wait_msec=%dms, sync_interval=%dms, " \ + "sync_start_time=%02d:%02d, sync_end_time=%02d:%02d, "\ + "write_mark_file_freq=%d, " \ + "allow_ip_count=%d, " \ + "file_distribute_path_mode=%d, " \ + "file_distribute_rotate_count=%d, " \ + "fsync_after_written_bytes=%d, " \ + "sync_log_buff_interval=%ds, " \ + "sync_binlog_buff_interval=%ds, " \ + "sync_stat_file_interval=%ds, " \ + "thread_stack_size=%d KB, upload_priority=%d, " \ + "if_alias_prefix=%s, " \ + "check_file_duplicate=%d, file_signature_method=%s, " \ + "FDHT group count=%d, FDHT server count=%d, " \ + "FDHT key_namespace=%s, FDHT keep_alive=%d, " \ + "HTTP server port=%d, domain name=%s, " \ + "use_access_log=%d, rotate_access_log=%d, " \ + "access_log_rotate_time=%02d:%02d, " \ + "rotate_error_log=%d, " \ + "error_log_rotate_time=%02d:%02d, " \ + "rotate_access_log_size="INT64_PRINTF_FORMAT", " \ + "rotate_error_log_size="INT64_PRINTF_FORMAT", " \ + "file_sync_skip_invalid_record=%d, " \ + "use_connection_pool=%d, " \ + "g_connection_pool_max_idle_time=%ds", \ + g_fdfs_version.major, g_fdfs_version.minor, \ + g_fdfs_base_path, g_fdfs_store_paths.count, \ + g_subdir_count_per_path, \ + g_group_name, g_run_by_group, g_run_by_user, \ + g_fdfs_connect_timeout, g_fdfs_network_timeout, \ + g_server_port, bind_addr, \ + g_client_bind_addr, g_max_connections, \ + g_accept_threads, g_work_threads, g_disk_rw_separated, \ + g_disk_reader_threads, g_disk_writer_threads, \ + g_buff_size / 1024, \ + g_heart_beat_interval, g_stat_report_interval, \ + g_tracker_group.server_count, g_sync_wait_usec / 1000, \ + g_sync_interval / 1000, \ + g_sync_start_time.hour, g_sync_start_time.minute, \ + g_sync_end_time.hour, g_sync_end_time.minute, \ + g_write_mark_file_freq, \ + g_allow_ip_count, g_file_distribute_path_mode, \ + g_file_distribute_rotate_count, \ + g_fsync_after_written_bytes, g_sync_log_buff_interval, \ + g_sync_binlog_buff_interval, g_sync_stat_file_interval, \ + g_thread_stack_size/1024, g_upload_priority, \ + g_if_alias_prefix, g_check_file_duplicate, \ + g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH \ + ? "hash" : "md5", + g_group_array.group_count, g_group_array.server_count, \ + g_key_namespace, g_keep_alive, \ + g_http_port, g_http_domain, g_use_access_log, \ + g_rotate_access_log, g_access_log_rotate_time.hour, \ + g_access_log_rotate_time.minute, \ + g_rotate_error_log, g_error_log_rotate_time.hour, \ + g_error_log_rotate_time.minute, \ + g_access_log_context.rotate_size, \ + g_log_context.rotate_size, \ + g_file_sync_skip_invalid_record, \ + g_use_connection_pool, g_connection_pool_max_idle_time); + +#ifdef WITH_HTTPD + if (!g_http_params.disabled) + { + logInfo("HTTP supported: " \ + "server_port=%d, " \ + "http_trunk_size=%d, " \ + "default_content_type=%s, " \ + "anti_steal_token=%d, " \ + "token_ttl=%ds, " \ + "anti_steal_secret_key length=%d, " \ + "token_check_fail content_type=%s, " \ + "token_check_fail buff length=%d", \ + g_http_params.server_port, \ + g_http_trunk_size, \ + g_http_params.default_content_type, \ + g_http_params.anti_steal_token, \ + g_http_params.token_ttl, \ + g_http_params.anti_steal_secret_key.length, \ + g_http_params.token_check_fail_content_type, \ + g_http_params.token_check_fail_buff.length); + } +#endif + + } while (0); + + iniFreeContext(&iniContext); + + if (result != 0) + { + return result; + } + + if ((result=storage_get_my_tracker_client_ip()) != 0) + { + return result; + } + + if ((result=storage_check_and_make_data_dirs()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_check_and_make_data_dirs fail, " \ + "program exit!", __LINE__); + return result; + } + + if ((result=storage_get_params_from_tracker()) != 0) + { + return result; + } + + if ((result=tracker_get_my_server_id()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "get my server id from tracker server fail, " \ + "errno: %d, error info: %s", __LINE__, \ + result, STRERROR(result)); + return result; + } + + if (g_use_storage_id) + { + if ((result=fdfs_get_storage_ids_from_tracker_group( \ + &g_tracker_group)) != 0) + { + return result; + } + } + + if ((result=storage_check_ip_changed()) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&sync_stat_file_lock)) != 0) + { + return result; + } + + return storage_open_stat_file(); +} + +int storage_func_destroy() +{ + int i; + int result; + int close_ret; + + if (g_fdfs_store_paths.paths != NULL) + { + for (i=0; iid, g_my_server_id_str) == 0; + } + else + { + return is_local_host_ip(pStorageBrief->ip_addr); + } +} + +bool storage_id_is_myself(const char *storage_id) +{ + if (g_use_storage_id) + { + return strcmp(storage_id, g_my_server_id_str) == 0; + } + else + { + return is_local_host_ip(storage_id); + } +} + +/* +int write_serialized(int fd, const char *buff, size_t count, const bool bSync) +{ + int result; + int fsync_ret; + + if ((result=pthread_mutex_lock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + while (fsync_thread_count >= g_max_write_thread_count) + { + if ((result=pthread_cond_wait(&fsync_thread_cond, \ + &fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "pthread_cond_wait failed, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + fsync_thread_count++; + + if ((result=pthread_mutex_unlock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if (write(fd, buff, count) == count) + { + if (bSync && fsync(fd) != 0) + { + fsync_ret = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "call fsync fail, " \ + "errno: %d, error info: %s", \ + __LINE__, fsync_ret, STRERROR(fsync_ret)); + } + else + { + fsync_ret = 0; + } + } + else + { + fsync_ret = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "call write fail, " \ + "errno: %d, error info: %s", \ + __LINE__, fsync_ret, STRERROR(fsync_ret)); + } + + if ((result=pthread_mutex_lock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + fsync_thread_count--; + + if ((result=pthread_mutex_unlock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if ((result=pthread_cond_signal(&fsync_thread_cond)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "pthread_cond_signal failed, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return fsync_ret; +} + +int fsync_serialized(int fd) +{ + int result; + int fsync_ret; + + if ((result=pthread_mutex_lock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + while (fsync_thread_count >= g_max_write_thread_count) + { + if ((result=pthread_cond_wait(&fsync_thread_cond, \ + &fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "pthread_cond_wait failed, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + fsync_thread_count++; + + if ((result=pthread_mutex_unlock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if (fsync(fd) == 0) + { + fsync_ret = 0; + } + else + { + fsync_ret = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "call fsync fail, " \ + "errno: %d, error info: %s", \ + __LINE__, fsync_ret, STRERROR(fsync_ret)); + } + + if ((result=pthread_mutex_lock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + fsync_thread_count--; + + if ((result=pthread_mutex_unlock(&fsync_thread_mutex)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if ((result=pthread_cond_signal(&fsync_thread_cond)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "pthread_cond_signal failed, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return fsync_ret; +} + +int recv_file_serialized(int sock, const char *filename, \ + const int64_t file_bytes) +{ + int fd; + char buff[FDFS_WRITE_BUFF_SIZE]; + int64_t remain_bytes; + int recv_bytes; + int result; + + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + { + return errno != 0 ? errno : EACCES; + } + + remain_bytes = file_bytes; + while (remain_bytes > 0) + { + if (remain_bytes > sizeof(buff)) + { + recv_bytes = sizeof(buff); + } + else + { + recv_bytes = remain_bytes; + } + + if ((result=tcprecvdata_nb(sock, buff, recv_bytes, \ + g_fdfs_network_timeout)) != 0) + { + close(fd); + unlink(filename); + return result; + } + + if (recv_bytes == remain_bytes) //last buff + { + if (write_serialized(fd, buff, recv_bytes, true) != 0) + { + result = errno != 0 ? errno : EIO; + close(fd); + unlink(filename); + return result; + } + } + else + { + if (write_serialized(fd, buff, recv_bytes, false) != 0) + { + result = errno != 0 ? errno : EIO; + close(fd); + unlink(filename); + return result; + } + + if ((result=fsync_serialized(fd)) != 0) + { + close(fd); + unlink(filename); + return result; + } + } + + remain_bytes -= recv_bytes; + } + + close(fd); + return 0; +} +*/ diff --git a/storage/storage_func.h b/storage/storage_func.h new file mode 100644 index 0000000..e959d2a --- /dev/null +++ b/storage/storage_func.h @@ -0,0 +1,78 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_func.h + +#ifndef _STORAGE_FUNC_H_ +#define _STORAGE_FUNC_H_ + +#include "tracker_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef char * (*get_filename_func)(const void *pArg, \ + char *full_filename); + +int storage_write_to_fd(int fd, get_filename_func filename_func, \ + const void *pArg, const char *buff, const int len); +int storage_func_init(const char *filename, \ + char *bind_addr, const int addr_size); +int storage_func_destroy(); + +int storage_write_to_stat_file(); + +int storage_write_to_sync_ini_file(); + +bool storage_server_is_myself(const FDFSStorageBrief *pStorageBrief); + +bool storage_id_is_myself(const char *storage_id); + +#define STORAGE_CHOWN(path, current_uid, current_gid) \ + if (!(g_run_by_gid == current_gid && g_run_by_uid == current_uid)) \ + { \ + if (chown(path, g_run_by_uid, g_run_by_gid) != 0) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "chown \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, path, \ + errno, STRERROR(errno)); \ + return errno != 0 ? errno : EPERM; \ + } \ + } + + +#define STORAGE_FCHOWN(fd, path, current_uid, current_gid) \ + if (!(g_run_by_gid == current_gid && g_run_by_uid == current_uid)) \ + { \ + if (fchown(fd, g_run_by_uid, g_run_by_gid) != 0) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "chown \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, path, \ + errno, STRERROR(errno)); \ + return errno != 0 ? errno : EPERM; \ + } \ + } + + +/* +int write_serialized(int fd, const char *buff, size_t count, const bool bSync); +int fsync_serialized(int fd); +int recv_file_serialized(int sock, const char *filename, \ + const int64_t file_bytes); +*/ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/storage_global.c b/storage/storage_global.c new file mode 100644 index 0000000..6a5954f --- /dev/null +++ b/storage/storage_global.c @@ -0,0 +1,132 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include "logger.h" +#include "sockopt.h" +#include "shared_func.h" +#include "storage_global.h" + +volatile bool g_continue_flag = true; +FDFSStorePathInfo *g_path_space_list = NULL; +int g_subdir_count_per_path = DEFAULT_DATA_DIR_COUNT_PER_PATH; + +int g_server_port = FDFS_STORAGE_SERVER_DEF_PORT; +char g_http_domain[FDFS_DOMAIN_NAME_MAX_SIZE] = {0}; +int g_http_port = 80; +int g_last_server_port = 0; +int g_last_http_port = 0; +int g_max_connections = DEFAULT_MAX_CONNECTONS; +int g_accept_threads = 1; +int g_work_threads = DEFAULT_WORK_THREADS; +int g_buff_size = STORAGE_DEFAULT_BUFF_SIZE; + +bool g_disk_rw_direct = false; +bool g_disk_rw_separated = true; +int g_disk_reader_threads = DEFAULT_DISK_READER_THREADS; +int g_disk_writer_threads = DEFAULT_DISK_WRITER_THREADS; +int g_extra_open_file_flags = 0; + +int g_file_distribute_path_mode = FDFS_FILE_DIST_PATH_ROUND_ROBIN; +int g_file_distribute_rotate_count = FDFS_FILE_DIST_DEFAULT_ROTATE_COUNT; +int g_fsync_after_written_bytes = -1; + +struct timeval g_network_tv = {DEFAULT_NETWORK_TIMEOUT, 0}; + +int g_dist_path_index_high = 0; //current write to high path +int g_dist_path_index_low = 0; //current write to low path +int g_dist_write_file_count = 0; //current write file count + +int g_storage_count = 0; +FDFSStorageServer g_storage_servers[FDFS_MAX_SERVERS_EACH_GROUP]; +FDFSStorageServer *g_sorted_storages[FDFS_MAX_SERVERS_EACH_GROUP]; + +int g_tracker_reporter_count = 0; +int g_heart_beat_interval = STORAGE_BEAT_DEF_INTERVAL; +int g_stat_report_interval = STORAGE_REPORT_DEF_INTERVAL; + +int g_sync_wait_usec = STORAGE_DEF_SYNC_WAIT_MSEC; +int g_sync_interval = 0; //unit: milliseconds +TimeInfo g_sync_start_time = {0, 0}; +TimeInfo g_sync_end_time = {23, 59}; +bool g_sync_part_time = false; +int g_sync_log_buff_interval = SYNC_LOG_BUFF_DEF_INTERVAL; +int g_sync_binlog_buff_interval = SYNC_BINLOG_BUFF_DEF_INTERVAL; +int g_write_mark_file_freq = FDFS_DEFAULT_SYNC_MARK_FILE_FREQ; +int g_sync_stat_file_interval = DEFAULT_SYNC_STAT_FILE_INTERVAL; + +FDFSStorageStat g_storage_stat; +int g_stat_change_count = 1; +int g_sync_change_count = 0; + +int g_storage_join_time = 0; +int g_sync_until_timestamp = 0; +bool g_sync_old_done = false; +char g_sync_src_id[FDFS_STORAGE_ID_MAX_SIZE] = {0}; + +char g_group_name[FDFS_GROUP_NAME_MAX_LEN + 1] = {0}; +char g_my_server_id_str[FDFS_STORAGE_ID_MAX_SIZE] = {0}; //my server id string +char g_tracker_client_ip[IP_ADDRESS_SIZE] = {0}; //storage ip as tracker client +char g_last_storage_ip[IP_ADDRESS_SIZE] = {0}; //the last storage ip address + +LogContext g_access_log_context = {LOG_INFO, STDERR_FILENO, NULL}; + +in_addr_t g_server_id_in_filename = 0; +bool g_use_access_log = false; //if log to access log +bool g_rotate_access_log = false; //if rotate the access log every day +bool g_rotate_error_log = false; //if rotate the error log every day +bool g_use_storage_id = false; //identify storage by ID instead of IP address +byte g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS; //id type of the storage server in the filename +bool g_store_slave_file_use_link = false; //if store slave file use symbol link + +bool g_check_file_duplicate = false; +byte g_file_signature_method = STORAGE_FILE_SIGNATURE_METHOD_HASH; +char g_key_namespace[FDHT_MAX_NAMESPACE_LEN+1] = {0}; +int g_namespace_len = 0; +int g_allow_ip_count = 0; +in_addr_t *g_allow_ip_addrs = NULL; + +TimeInfo g_access_log_rotate_time = {0, 0}; //rotate access log time base +TimeInfo g_error_log_rotate_time = {0, 0}; //rotate error log time base + +gid_t g_run_by_gid; +uid_t g_run_by_uid; + +char g_run_by_group[32] = {0}; +char g_run_by_user[32] = {0}; + +char g_bind_addr[IP_ADDRESS_SIZE] = {0}; +bool g_client_bind_addr = true; +bool g_storage_ip_changed_auto_adjust = false; +bool g_thread_kill_done = false; +bool g_file_sync_skip_invalid_record = false; + +int g_thread_stack_size = 512 * 1024; +int g_upload_priority = DEFAULT_UPLOAD_PRIORITY; +time_t g_up_time = 0; + +#ifdef WITH_HTTPD +FDFSHTTPParams g_http_params; +int g_http_trunk_size = 64 * 1024; +#endif + +#if defined(DEBUG_FLAG) && defined(OS_LINUX) +char g_exe_name[256] = {0}; +#endif + +struct storage_nio_thread_data *g_nio_thread_data = NULL; +struct storage_dio_thread_data *g_dio_thread_data = NULL; + +int storage_cmp_by_server_id(const void *p1, const void *p2) +{ + return strcmp((*((FDFSStorageServer **)p1))->server.id, + (*((FDFSStorageServer **)p2))->server.id); +} + diff --git a/storage/storage_global.h b/storage/storage_global.h new file mode 100644 index 0000000..2c180eb --- /dev/null +++ b/storage/storage_global.h @@ -0,0 +1,181 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_global.h + +#ifndef _STORAGE_GLOBAL_H +#define _STORAGE_GLOBAL_H + +#include +#include +#include +#include +#include "common_define.h" +#include "fdfs_define.h" +#include "tracker_types.h" +#include "client_global.h" +#include "fdht_types.h" +#include "local_ip_func.h" +#include "trunk_shared.h" + +#ifdef WITH_HTTPD +#include "fdfs_http_shared.h" +#endif + +#define STORAGE_BEAT_DEF_INTERVAL 30 +#define STORAGE_REPORT_DEF_INTERVAL 300 +#define STORAGE_DEF_SYNC_WAIT_MSEC 100 +#define DEFAULT_DISK_READER_THREADS 1 +#define DEFAULT_DISK_WRITER_THREADS 1 +#define DEFAULT_SYNC_STAT_FILE_INTERVAL 300 +#define DEFAULT_DATA_DIR_COUNT_PER_PATH 256 +#define DEFAULT_UPLOAD_PRIORITY 10 +#define FDFS_DEFAULT_SYNC_MARK_FILE_FREQ 500 +#define STORAGE_DEFAULT_BUFF_SIZE (64 * 1024) + +#define STORAGE_FILE_SIGNATURE_METHOD_HASH 1 +#define STORAGE_FILE_SIGNATURE_METHOD_MD5 2 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct +{ + FDFSStorageBrief server; + int last_sync_src_timestamp; +} FDFSStorageServer; + +typedef struct +{ + int total_mb; //total spaces + int free_mb; //free spaces +} FDFSStorePathInfo; + +extern volatile bool g_continue_flag; + +extern FDFSStorePathInfo *g_path_space_list; + +/* subdirs under store path, g_subdir_count * g_subdir_count 2 level subdirs */ +extern int g_subdir_count_per_path; + +extern int g_server_port; +extern int g_http_port; //http server port +extern int g_last_server_port; +extern int g_last_http_port; //last http server port +extern char g_http_domain[FDFS_DOMAIN_NAME_MAX_SIZE]; //http server domain name +extern int g_max_connections; +extern int g_accept_threads; +extern int g_work_threads; +extern int g_buff_size; + +extern bool g_disk_rw_direct; //if file read / write directly +extern bool g_disk_rw_separated; //if disk read / write separated +extern int g_disk_reader_threads; //disk reader thread count per store base path +extern int g_disk_writer_threads; //disk writer thread count per store base path +extern int g_extra_open_file_flags; //extra open file flags + +extern int g_file_distribute_path_mode; +extern int g_file_distribute_rotate_count; +extern int g_fsync_after_written_bytes; + +extern struct timeval g_network_tv; + +extern int g_dist_path_index_high; //current write to high path +extern int g_dist_path_index_low; //current write to low path +extern int g_dist_write_file_count; //current write file count + +extern int g_storage_count; //stoage server count in my group +extern FDFSStorageServer g_storage_servers[FDFS_MAX_SERVERS_EACH_GROUP]; +extern FDFSStorageServer *g_sorted_storages[FDFS_MAX_SERVERS_EACH_GROUP]; + +extern int g_tracker_reporter_count; +extern int g_heart_beat_interval; +extern int g_stat_report_interval; +extern int g_sync_wait_usec; +extern int g_sync_interval; //unit: milliseconds +extern TimeInfo g_sync_start_time; +extern TimeInfo g_sync_end_time; +extern bool g_sync_part_time; //true for part time, false for all time of a day +extern int g_sync_log_buff_interval; //sync log buff to disk every interval seconds +extern int g_sync_binlog_buff_interval; //sync binlog buff to disk every interval seconds +extern int g_write_mark_file_freq; //write to mark file after sync N files +extern int g_sync_stat_file_interval; //sync storage stat info to disk interval + +extern FDFSStorageStat g_storage_stat; +extern int g_stat_change_count; +extern int g_sync_change_count; //sync src timestamp change counter + +extern int g_storage_join_time; //my join timestamp +extern int g_sync_until_timestamp; +extern bool g_sync_old_done; //if old files synced to me done +extern char g_sync_src_id[FDFS_STORAGE_ID_MAX_SIZE]; //the source storage server id + +extern char g_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; +extern char g_my_server_id_str[FDFS_STORAGE_ID_MAX_SIZE]; //my server id string +extern char g_tracker_client_ip[IP_ADDRESS_SIZE]; //storage ip as tracker client +extern char g_last_storage_ip[IP_ADDRESS_SIZE]; //the last storage ip address + +extern LogContext g_access_log_context; + +extern in_addr_t g_server_id_in_filename; +extern bool g_store_slave_file_use_link; //if store slave file use symbol link +extern bool g_use_storage_id; //identify storage by ID instead of IP address +extern byte g_id_type_in_filename; //id type of the storage server in the filename +extern bool g_use_access_log; //if log to access log +extern bool g_rotate_access_log; //if rotate the access log every day +extern bool g_rotate_error_log; //if rotate the error log every day + +extern TimeInfo g_access_log_rotate_time; //rotate access log time base +extern TimeInfo g_error_log_rotate_time; //rotate error log time base + +extern bool g_check_file_duplicate; //if check file content duplicate +extern byte g_file_signature_method; //file signature method +extern char g_key_namespace[FDHT_MAX_NAMESPACE_LEN+1]; +extern int g_namespace_len; + +extern int g_allow_ip_count; /* -1 means match any ip address */ +extern in_addr_t *g_allow_ip_addrs; /* sorted array, asc order */ + +extern gid_t g_run_by_gid; +extern uid_t g_run_by_uid; + +extern char g_run_by_group[32]; +extern char g_run_by_user[32]; + +extern char g_bind_addr[IP_ADDRESS_SIZE]; +extern bool g_client_bind_addr; +extern bool g_storage_ip_changed_auto_adjust; +extern bool g_thread_kill_done; + +extern bool g_file_sync_skip_invalid_record; + +extern int g_thread_stack_size; +extern int g_upload_priority; +extern time_t g_up_time; + +#ifdef WITH_HTTPD +extern FDFSHTTPParams g_http_params; +extern int g_http_trunk_size; +#endif + +#if defined(DEBUG_FLAG) && defined(OS_LINUX) +extern char g_exe_name[256]; +#endif + +extern struct storage_nio_thread_data *g_nio_thread_data; //network io thread data +extern struct storage_dio_thread_data *g_dio_thread_data; //disk io thread data + +int storage_cmp_by_server_id(const void *p1, const void *p2); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/storage_ip_changed_dealer.c b/storage/storage_ip_changed_dealer.c new file mode 100644 index 0000000..5b07282 --- /dev/null +++ b/storage/storage_ip_changed_dealer.c @@ -0,0 +1,440 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_ip_changed_dealer.h" + +static int storage_do_changelog_req(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_STORAGE_ID_MAX_SIZE]; + TrackerHeader *pHeader; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + + long2buff(FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE, \ + pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ; + strcpy(out_buff + sizeof(TrackerHeader), g_group_name); + strcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN, + g_my_server_id_str); + if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + return tracker_deal_changelog_response(pTrackerServer); +} + +static int storage_report_ip_changed(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + 2 * IP_ADDRESS_SIZE]; + char in_buff[1]; + char *pInBuff; + TrackerHeader *pHeader; + int result; + int64_t in_bytes; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + + long2buff(FDFS_GROUP_NAME_MAX_LEN+2*IP_ADDRESS_SIZE, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_IP_CHANGED; + strcpy(out_buff + sizeof(TrackerHeader), g_group_name); + strcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN, \ + g_last_storage_ip); + strcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + IP_ADDRESS_SIZE, g_tracker_client_ip); + + if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result; + } + + pInBuff = in_buff; + result = fdfs_recv_response(pTrackerServer, \ + &pInBuff, 0, &in_bytes); + + if (result == 0 || result == EALREADY || result == ENOENT) + { + return 0; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data fail or " \ + "response status != 0, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result == EBUSY ? 0 : result; + } +} + +int storage_get_my_tracker_client_ip() +{ + ConnectionInfo *pGlobalServer; + ConnectionInfo *pTServer; + ConnectionInfo *pTServerEnd; + ConnectionInfo trackerServer; + char tracker_client_ip[IP_ADDRESS_SIZE]; + int success_count; + int result; + int i; + + result = 0; + success_count = 0; + pTServer = &trackerServer; + pTServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + + while (success_count == 0 && g_continue_flag) + { + for (pGlobalServer=g_tracker_group.servers; pGlobalServersock = socket(AF_INET, SOCK_STREAM, 0); + if(pTServer->sock < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s.", \ + __LINE__, result, STRERROR(result)); + sleep(5); + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(pTServer->sock, g_bind_addr, 0); + } + + if (tcpsetnonblockopt(pTServer->sock) != 0) + { + close(pTServer->sock); + pTServer->sock = -1; + sleep(5); + continue; + } + + if ((result=connectserverbyip_nb(pTServer->sock, \ + pTServer->ip_addr, pTServer->port, \ + g_fdfs_connect_timeout)) == 0) + { + break; + } + + close(pTServer->sock); + pTServer->sock = -1; + sleep(5); + } + + if (pTServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTServer->ip_addr, pTServer->port, \ + result, STRERROR(result)); + + continue; + } + + getSockIpaddr(pTServer->sock,tracker_client_ip,IP_ADDRESS_SIZE); + if (*g_tracker_client_ip == '\0') + { + strcpy(g_tracker_client_ip, tracker_client_ip); + } + else if (strcmp(tracker_client_ip, g_tracker_client_ip) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "as a client of tracker server %s:%d, " \ + "my ip: %s != client ip: %s of other " \ + "tracker client", __LINE__, \ + pTServer->ip_addr, pTServer->port, \ + tracker_client_ip, g_tracker_client_ip); + + close(pTServer->sock); + return EINVAL; + } + + fdfs_quit(pTServer); + close(pTServer->sock); + success_count++; + } + } + + if (!g_continue_flag) + { + return EINTR; + } + + return 0; +} + +static int storage_report_storage_ip_addr() +{ + ConnectionInfo *pGlobalServer; + ConnectionInfo *pTServer; + ConnectionInfo *pTServerEnd; + ConnectionInfo trackerServer; + int success_count; + int result; + int i; + + result = 0; + success_count = 0; + pTServer = &trackerServer; + pTServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + + logDebug("file: "__FILE__", line: %d, " \ + "last my ip is %s, current my ip is %s", \ + __LINE__, g_last_storage_ip, g_tracker_client_ip); + + if (*g_last_storage_ip == '\0') + { + return storage_write_to_sync_ini_file(); + } + else if (strcmp(g_tracker_client_ip, g_last_storage_ip) == 0) + { + return 0; + } + + success_count = 0; + while (success_count == 0 && g_continue_flag) + { + for (pGlobalServer=g_tracker_group.servers; pGlobalServersock = socket(AF_INET, SOCK_STREAM, 0); + if(pTServer->sock < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s.", \ + __LINE__, result, STRERROR(result)); + sleep(5); + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(pTServer->sock, g_bind_addr, 0); + } + + if (tcpsetnonblockopt(pTServer->sock) != 0) + { + close(pTServer->sock); + pTServer->sock = -1; + sleep(1); + continue; + } + + if ((result=connectserverbyip_nb(pTServer->sock, \ + pTServer->ip_addr, pTServer->port, \ + g_fdfs_connect_timeout)) == 0) + { + break; + } + + close(pTServer->sock); + pTServer->sock = -1; + sleep(1); + } + + if (pTServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTServer->ip_addr, pTServer->port, \ + result, STRERROR(result)); + + continue; + } + + if ((result=storage_report_ip_changed(pTServer)) == 0) + { + success_count++; + } + else + { + sleep(1); + } + + fdfs_quit(pTServer); + close(pTServer->sock); + } + } + + if (!g_continue_flag) + { + return EINTR; + } + + return storage_write_to_sync_ini_file(); +} + +int storage_changelog_req() +{ + ConnectionInfo *pGlobalServer; + ConnectionInfo *pTServer; + ConnectionInfo *pTServerEnd; + ConnectionInfo trackerServer; + int success_count; + int result; + int i; + + result = 0; + success_count = 0; + pTServer = &trackerServer; + pTServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + + while (success_count == 0 && g_continue_flag) + { + for (pGlobalServer=g_tracker_group.servers; pGlobalServersock = socket(AF_INET, SOCK_STREAM, 0); + if(pTServer->sock < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s.", \ + __LINE__, result, STRERROR(result)); + sleep(5); + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(pTServer->sock, g_bind_addr, 0); + } + + if (tcpsetnonblockopt(pTServer->sock) != 0) + { + close(pTServer->sock); + pTServer->sock = -1; + sleep(1); + continue; + } + + if ((result=connectserverbyip_nb(pTServer->sock, \ + pTServer->ip_addr, pTServer->port, \ + g_fdfs_connect_timeout)) == 0) + { + break; + } + + close(pTServer->sock); + pTServer->sock = -1; + sleep(1); + } + + if (pTServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTServer->ip_addr, pTServer->port, \ + result, STRERROR(result)); + + continue; + } + + result = storage_do_changelog_req(pTServer); + if (result == 0 || result == ENOENT) + { + success_count++; + } + else + { + sleep(1); + } + + fdfs_quit(pTServer); + close(pTServer->sock); + } + } + + if (!g_continue_flag) + { + return EINTR; + } + + return 0; +} + +int storage_check_ip_changed() +{ + int result; + + if ((!g_storage_ip_changed_auto_adjust) || g_use_storage_id) + { + return 0; + } + + if ((result=storage_report_storage_ip_addr()) != 0) + { + return result; + } + + if (*g_last_storage_ip == '\0') //first run + { + return 0; + } + + return storage_changelog_req(); +} + diff --git a/storage/storage_ip_changed_dealer.h b/storage/storage_ip_changed_dealer.h new file mode 100644 index 0000000..f5639ec --- /dev/null +++ b/storage/storage_ip_changed_dealer.h @@ -0,0 +1,31 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_ip_changed_dealer.h + +#ifndef _STORAGE_IP_CHANGED_DEALER_H_ +#define _STORAGE_IP_CHANGED_DEALER_H_ + +#include "tracker_types.h" +#include "tracker_client_thread.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int storage_get_my_tracker_client_ip(); + +int storage_changelog_req(); +int storage_check_ip_changed(); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/storage_nio.c b/storage/storage_nio.c new file mode 100644 index 0000000..2cfcc2b --- /dev/null +++ b/storage/storage_nio.c @@ -0,0 +1,524 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "sched_thread.h" +#include "logger.h" +#include "sockopt.h" +#include "fast_task_queue.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "storage_service.h" +#include "ioevent_loop.h" +#include "storage_dio.h" +#include "storage_nio.h" + +static void client_sock_read(int sock, short event, void *arg); +static void client_sock_write(int sock, short event, void *arg); +static int storage_nio_init(struct fast_task_info *pTask); + +void add_to_deleted_list(struct fast_task_info *pTask) +{ + ((StorageClientInfo *)pTask->arg)->canceled = true; + pTask->next = pTask->thread_data->deleted_list; + pTask->thread_data->deleted_list = pTask; +} + +void task_finish_clean_up(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + + pClientInfo = (StorageClientInfo *)pTask->arg; + if (pClientInfo->clean_func != NULL) + { + pClientInfo->clean_func(pTask); + } + + ioevent_detach(&pTask->thread_data->ev_puller, pTask->event.fd); + close(pTask->event.fd); + pTask->event.fd = -1; + + if (pTask->event.timer.expires > 0) + { + fast_timer_remove(&pTask->thread_data->timer, + &pTask->event.timer); + pTask->event.timer.expires = 0; + } + + memset(pTask->arg, 0, sizeof(StorageClientInfo)); + free_queue_push(pTask); +} + +static int set_recv_event(struct fast_task_info *pTask) +{ + int result; + + if (pTask->event.callback == client_sock_read) + { + return 0; + } + + pTask->event.callback = client_sock_read; + if (ioevent_modify(&pTask->thread_data->ev_puller, + pTask->event.fd, IOEVENT_READ, pTask) != 0) + { + result = errno != 0 ? errno : ENOENT; + add_to_deleted_list(pTask); + + logError("file: "__FILE__", line: %d, "\ + "ioevent_modify fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + return 0; +} + +static int set_send_event(struct fast_task_info *pTask) +{ + int result; + + if (pTask->event.callback == client_sock_write) + { + return 0; + } + + pTask->event.callback = client_sock_write; + if (ioevent_modify(&pTask->thread_data->ev_puller, + pTask->event.fd, IOEVENT_WRITE, pTask) != 0) + { + result = errno != 0 ? errno : ENOENT; + add_to_deleted_list(pTask); + + logError("file: "__FILE__", line: %d, "\ + "ioevent_modify fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + return 0; +} + +void storage_recv_notify_read(int sock, short event, void *arg) +{ + struct fast_task_info *pTask; + StorageClientInfo *pClientInfo; + long task_addr; + int64_t remain_bytes; + int bytes; + int result; + + while (1) + { + if ((bytes=read(sock, &task_addr, sizeof(task_addr))) < 0) + { + if (!(errno == EAGAIN || errno == EWOULDBLOCK)) + { + logError("file: "__FILE__", line: %d, " \ + "call read failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + + break; + } + else if (bytes == 0) + { + logError("file: "__FILE__", line: %d, " \ + "call read failed, end of file", __LINE__); + break; + } + + pTask = (struct fast_task_info *)task_addr; + pClientInfo = (StorageClientInfo *)pTask->arg; + + if (pTask->event.fd < 0) //quit flag + { + return; + } + + /* //logInfo("=====thread index: %d, pTask->event.fd=%d", \ + pClientInfo->nio_thread_index, pTask->event.fd); + */ + + if (pClientInfo->stage & FDFS_STORAGE_STAGE_DIO_THREAD) + { + pClientInfo->stage &= ~FDFS_STORAGE_STAGE_DIO_THREAD; + } + switch (pClientInfo->stage) + { + case FDFS_STORAGE_STAGE_NIO_INIT: + result = storage_nio_init(pTask); + break; + case FDFS_STORAGE_STAGE_NIO_RECV: + pTask->offset = 0; + remain_bytes = pClientInfo->total_length - \ + pClientInfo->total_offset; + if (remain_bytes > pTask->size) + { + pTask->length = pTask->size; + } + else + { + pTask->length = remain_bytes; + } + + if (set_recv_event(pTask) == 0) + { + client_sock_read(pTask->event.fd, + IOEVENT_READ, pTask); + } + result = 0; + break; + case FDFS_STORAGE_STAGE_NIO_SEND: + result = storage_send_add_event(pTask); + break; + case FDFS_STORAGE_STAGE_NIO_CLOSE: + result = EIO; //close this socket + break; + default: + logError("file: "__FILE__", line: %d, " \ + "invalid stage: %d", __LINE__, \ + pClientInfo->stage); + result = EINVAL; + break; + } + + if (result != 0) + { + add_to_deleted_list(pTask); + } + } +} + +static int storage_nio_init(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + struct storage_nio_thread_data *pThreadData; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pThreadData = g_nio_thread_data + pClientInfo->nio_thread_index; + + pClientInfo->stage = FDFS_STORAGE_STAGE_NIO_RECV; + return ioevent_set(pTask, &pThreadData->thread_data, + pTask->event.fd, IOEVENT_READ, client_sock_read, + g_fdfs_network_timeout); +} + +int storage_send_add_event(struct fast_task_info *pTask) +{ + pTask->offset = 0; + + /* direct send */ + client_sock_write(pTask->event.fd, IOEVENT_WRITE, pTask); + + return 0; +} + +static void client_sock_read(int sock, short event, void *arg) +{ + int bytes; + int recv_bytes; + struct fast_task_info *pTask; + StorageClientInfo *pClientInfo; + + pTask = (struct fast_task_info *)arg; + pClientInfo = (StorageClientInfo *)pTask->arg; + if (pClientInfo->canceled) + { + return; + } + + if (pClientInfo->stage != FDFS_STORAGE_STAGE_NIO_RECV) + { + if (event & IOEVENT_TIMEOUT) { + pTask->event.timer.expires = g_current_time + + g_fdfs_network_timeout; + fast_timer_add(&pTask->thread_data->timer, + &pTask->event.timer); + } + + return; + } + + if (event & IOEVENT_TIMEOUT) + { + if (pClientInfo->total_offset == 0 && pTask->req_count > 0) + { + pTask->event.timer.expires = g_current_time + + g_fdfs_network_timeout; + fast_timer_add(&pTask->thread_data->timer, + &pTask->event.timer); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv timeout, " \ + "recv offset: %d, expect length: %d", \ + __LINE__, pTask->client_ip, \ + pTask->offset, pTask->length); + + task_finish_clean_up(pTask); + } + + return; + } + + if (event & IOEVENT_ERROR) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv error event: %d, " + "close connection", __LINE__, pTask->client_ip, event); + + task_finish_clean_up(pTask); + return; + } + + fast_timer_modify(&pTask->thread_data->timer, + &pTask->event.timer, g_current_time + + g_fdfs_network_timeout); + while (1) + { + if (pClientInfo->total_length == 0) //recv header + { + recv_bytes = sizeof(TrackerHeader) - pTask->offset; + } + else + { + recv_bytes = pTask->length - pTask->offset; + } + + /* + logInfo("total_length="INT64_PRINTF_FORMAT", recv_bytes=%d, " + "pTask->length=%d, pTask->offset=%d", + pClientInfo->total_length, recv_bytes, + pTask->length, pTask->offset); + */ + + bytes = recv(sock, pTask->data + pTask->offset, recv_bytes, 0); + if (bytes < 0) + { + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv failed, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + errno, STRERROR(errno)); + + task_finish_clean_up(pTask); + } + + return; + } + else if (bytes == 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "client ip: %s, recv failed, " \ + "connection disconnected.", \ + __LINE__, pTask->client_ip); + + task_finish_clean_up(pTask); + return; + } + + if (pClientInfo->total_length == 0) //header + { + if (pTask->offset + bytes < sizeof(TrackerHeader)) + { + pTask->offset += bytes; + return; + } + + pClientInfo->total_length=buff2long(((TrackerHeader *) \ + pTask->data)->pkg_len); + if (pClientInfo->total_length < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length: " \ + INT64_PRINTF_FORMAT" < 0", \ + __LINE__, pTask->client_ip, \ + pClientInfo->total_length); + + task_finish_clean_up(pTask); + return; + } + + pClientInfo->total_length += sizeof(TrackerHeader); + if (pClientInfo->total_length > pTask->size) + { + pTask->length = pTask->size; + } + else + { + pTask->length = pClientInfo->total_length; + } + } + + pTask->offset += bytes; + if (pTask->offset >= pTask->length) //recv current pkg done + { + if (pClientInfo->total_offset + pTask->length >= \ + pClientInfo->total_length) + { + /* current req recv done */ + pClientInfo->stage = FDFS_STORAGE_STAGE_NIO_SEND; + pTask->req_count++; + } + + if (pClientInfo->total_offset == 0) + { + pClientInfo->total_offset = pTask->length; + storage_deal_task(pTask); + } + else + { + pClientInfo->total_offset += pTask->length; + + /* continue write to file */ + storage_dio_queue_push(pTask); + } + + return; + } + } + + return; +} + +static void client_sock_write(int sock, short event, void *arg) +{ + int bytes; + struct fast_task_info *pTask; + StorageClientInfo *pClientInfo; + + pTask = (struct fast_task_info *)arg; + pClientInfo = (StorageClientInfo *)pTask->arg; + if (pClientInfo->canceled) + { + return; + } + + if (event & IOEVENT_TIMEOUT) + { + logError("file: "__FILE__", line: %d, " \ + "send timeout", __LINE__); + + task_finish_clean_up(pTask); + return; + } + + if (event & IOEVENT_ERROR) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv error event: %d, " + "close connection", __LINE__, pTask->client_ip, event); + + task_finish_clean_up(pTask); + return; + } + + while (1) + { + fast_timer_modify(&pTask->thread_data->timer, + &pTask->event.timer, g_current_time + + g_fdfs_network_timeout); + bytes = send(sock, pTask->data + pTask->offset, \ + pTask->length - pTask->offset, 0); + //printf("%08X sended %d bytes\n", (int)pTask, bytes); + if (bytes < 0) + { + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + set_send_event(pTask); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv failed, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + errno, STRERROR(errno)); + + task_finish_clean_up(pTask); + } + + return; + } + else if (bytes == 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "send failed, connection disconnected.", \ + __LINE__); + + task_finish_clean_up(pTask); + return; + } + + pTask->offset += bytes; + if (pTask->offset >= pTask->length) + { + if (set_recv_event(pTask) != 0) + { + return; + } + + pClientInfo->total_offset += pTask->length; + if (pClientInfo->total_offset>=pClientInfo->total_length) + { + if (pClientInfo->total_length == sizeof(TrackerHeader) + && ((TrackerHeader *)pTask->data)->status == EINVAL) + { + logDebug("file: "__FILE__", line: %d, "\ + "close conn: #%d, client ip: %s", \ + __LINE__, pTask->event.fd, + pTask->client_ip); + task_finish_clean_up(pTask); + return; + } + + /* reponse done, try to recv again */ + pClientInfo->total_length = 0; + pClientInfo->total_offset = 0; + pTask->offset = 0; + pTask->length = 0; + + pClientInfo->stage = FDFS_STORAGE_STAGE_NIO_RECV; + } + else //continue to send file content + { + pTask->length = 0; + + /* continue read from file */ + storage_dio_queue_push(pTask); + } + + return; + } + } +} + diff --git a/storage/storage_nio.h b/storage/storage_nio.h new file mode 100644 index 0000000..7a14458 --- /dev/null +++ b/storage/storage_nio.h @@ -0,0 +1,159 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_nio.h + +#ifndef _TRACKER_NIO_H +#define _TRACKER_NIO_H + +#include +#include +#include +#include +#include "tracker_types.h" +#include "storage_func.h" +#include "fast_task_queue.h" +#include "storage_global.h" +#include "fdht_types.h" +#include "trunk_mem.h" +#include "md5.h" + +#define FDFS_STORAGE_STAGE_NIO_INIT 0 +#define FDFS_STORAGE_STAGE_NIO_RECV 1 +#define FDFS_STORAGE_STAGE_NIO_SEND 2 +#define FDFS_STORAGE_STAGE_NIO_CLOSE 4 //close socket +#define FDFS_STORAGE_STAGE_DIO_THREAD 8 + +#define FDFS_STORAGE_FILE_OP_READ 'R' +#define FDFS_STORAGE_FILE_OP_WRITE 'W' +#define FDFS_STORAGE_FILE_OP_APPEND 'A' +#define FDFS_STORAGE_FILE_OP_DELETE 'D' +#define FDFS_STORAGE_FILE_OP_DISCARD 'd' + +typedef int (*TaskDealFunc)(struct fast_task_info *pTask); + +/* this clean func will be called when connection disconnected */ +typedef void (*DisconnectCleanFunc)(struct fast_task_info *pTask); + +typedef void (*DeleteFileLogCallback)(struct fast_task_info *pTask, \ + const int err_no); + +typedef void (*FileDealDoneCallback)(struct fast_task_info *pTask, \ + const int err_no); + +typedef int (*FileBeforeOpenCallback)(struct fast_task_info *pTask); +typedef int (*FileBeforeCloseCallback)(struct fast_task_info *pTask); + +#define _FILE_TYPE_APPENDER 1 +#define _FILE_TYPE_TRUNK 2 //if trunk file, since V3.0 +#define _FILE_TYPE_SLAVE 4 +#define _FILE_TYPE_REGULAR 8 +#define _FILE_TYPE_LINK 16 + +typedef struct +{ + bool if_gen_filename; //if upload generate filename + char file_type; //regular or link file + bool if_sub_path_alloced; //if sub path alloced since V3.0 + char master_filename[128]; + char file_ext_name[FDFS_FILE_EXT_NAME_MAX_LEN + 1]; + char formatted_ext_name[FDFS_FILE_EXT_NAME_MAX_LEN + 2]; + char prefix_name[FDFS_FILE_PREFIX_MAX_LEN + 1]; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; //the upload group name + int start_time; //upload start timestamp + FDFSTrunkFullInfo trunk_info; + FileBeforeOpenCallback before_open_callback; + FileBeforeCloseCallback before_close_callback; +} StorageUploadInfo; + +typedef struct +{ + char op_flag; + char *meta_buff; + int meta_bytes; +} StorageSetMetaInfo; + +typedef struct +{ + char filename[MAX_PATH_SIZE + 128]; //full filename + + /* FDFS logic filename to log not including group name */ + char fname2log[128+sizeof(FDFS_STORAGE_META_FILE_EXT)]; + + char op; //w for writing, r for reading, d for deleting etc. + char sync_flag; //sync flag log to binlog + bool calc_crc32; //if calculate file content hash code + bool calc_file_hash; //if calculate file content hash code + int open_flags; //open file flags + int file_hash_codes[4]; //file hash code + int crc32; //file content crc32 signature + MD5_CTX md5_context; + + union + { + StorageUploadInfo upload; + StorageSetMetaInfo setmeta; + } extra_info; + + int dio_thread_index; //dio thread index + int timestamp2log; //timestamp to log + int delete_flag; //delete file flag + int create_flag; //create file flag + int buff_offset; //buffer offset after recv to write to file + int fd; //file description no + int64_t start; //the start offset of file + int64_t end; //the end offset of file + int64_t offset; //the current offset of file + FileDealDoneCallback done_callback; + DeleteFileLogCallback log_callback; + + struct timeval tv_deal_start; //task deal start tv for access log +} StorageFileContext; + +typedef struct +{ + int nio_thread_index; //nio thread index + bool canceled; + char stage; //nio stage, send or recv + char storage_server_id[FDFS_STORAGE_ID_MAX_SIZE]; + + StorageFileContext file_context; + + int64_t total_length; //pkg total length for req and request + int64_t total_offset; //pkg current offset for req and request + + int64_t request_length; //request pkg length for access log + + FDFSStorageServer *pSrcStorage; + TaskDealFunc deal_func; //function pointer to deal this task + void *extra_arg; //store extra arg, such as (BinLogReader *) + DisconnectCleanFunc clean_func; //clean function pointer when finished +} StorageClientInfo; + +struct storage_nio_thread_data +{ + struct nio_thread_data thread_data; + GroupArray group_array; //FastDHT group array +}; + +#ifdef __cplusplus +extern "C" { +#endif + +void storage_recv_notify_read(int sock, short event, void *arg); +int storage_send_add_event(struct fast_task_info *pTask); + +void task_finish_clean_up(struct fast_task_info *pTask); +void add_to_deleted_list(struct fast_task_info *pTask); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/storage_param_getter.c b/storage/storage_param_getter.c new file mode 100644 index 0000000..603e682 --- /dev/null +++ b/storage/storage_param_getter.c @@ -0,0 +1,232 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "shared_func.h" +#include "fdfs_global.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_client.h" +#include "fdfs_shared_func.h" +#include "storage_global.h" +#include "storage_param_getter.h" +#include "trunk_mem.h" +#include "trunk_sync.h" + +static int storage_convert_src_server_id() +{ + ConnectionInfo *pTrackerServer; + ConnectionInfo *pServerEnd; + ConnectionInfo *pTrackerConn; + ConnectionInfo tracker_server; + int result; + + result = ENOENT; + pServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + for (pTrackerServer=g_tracker_group.servers; \ + pTrackerServer +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_service.h" +#include "storage_func.h" +#include "storage_sync.h" +#include "storage_global.h" +#include "base64.h" +#include "hash.h" +#include "fdht_client.h" +#include "fdfs_global.h" +#include "tracker_client.h" +#include "storage_client.h" +#include "storage_nio.h" +#include "storage_dio.h" +#include "storage_sync.h" +#include "trunk_mem.h" +#include "trunk_sync.h" +#include "trunk_client.h" +#include "ioevent_loop.h" + +//storage access log actions +#define ACCESS_LOG_ACTION_UPLOAD_FILE "upload" +#define ACCESS_LOG_ACTION_DOWNLOAD_FILE "download" +#define ACCESS_LOG_ACTION_DELETE_FILE "delete" +#define ACCESS_LOG_ACTION_GET_METADATA "get_metadata" +#define ACCESS_LOG_ACTION_SET_METADATA "set_metadata" +#define ACCESS_LOG_ACTION_MODIFY_FILE "modify" +#define ACCESS_LOG_ACTION_APPEND_FILE "append" +#define ACCESS_LOG_ACTION_TRUNCATE_FILE "truncate" +#define ACCESS_LOG_ACTION_QUERY_FILE "status" + + +pthread_mutex_t g_storage_thread_lock; +int g_storage_thread_count = 0; + +static int last_stat_change_count = 1; //for sync to stat file +static int64_t temp_file_sequence = 0; + +static pthread_mutex_t path_index_thread_lock; +static pthread_mutex_t stat_count_thread_lock; + +static void *work_thread_entrance(void* arg); + +extern int storage_client_create_link(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pStorageServer, const char *master_filename,\ + const char *src_filename, const int src_filename_len, \ + const char *src_file_sig, const int src_file_sig_len, \ + const char *group_name, const char *prefix_name, \ + const char *file_ext_name, \ + char *remote_filename, int *filename_len); + +static int storage_do_delete_file(struct fast_task_info *pTask, \ + DeleteFileLogCallback log_callback, \ + FileDealDoneCallback done_callback, \ + const int store_path_index); + +static int storage_write_to_file(struct fast_task_info *pTask, \ + const int64_t file_offset, const int64_t upload_bytes, \ + const int buff_offset, TaskDealFunc deal_func, \ + FileDealDoneCallback done_callback, \ + DisconnectCleanFunc clean_func, const int store_path_index); + +static int storage_read_from_file(struct fast_task_info *pTask, \ + const int64_t file_offset, const int64_t download_bytes, \ + FileDealDoneCallback done_callback, \ + const int store_path_index); + +static int storage_service_upload_file_done(struct fast_task_info *pTask); + +#define STORAGE_STATUE_DEAL_FILE 123456 //status for read or write file + +#define FDHT_KEY_NAME_FILE_ID "fid" +#define FDHT_KEY_NAME_REF_COUNT "ref" +#define FDHT_KEY_NAME_FILE_SIG "sig" + +#define FILE_SIGNATURE_SIZE 24 + +#define STORAGE_GEN_FILE_SIGNATURE(file_size, hash_codes, sig_buff) \ + long2buff(file_size, sig_buff); \ + if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH) \ + { \ + int2buff(hash_codes[0], sig_buff + 8); \ + int2buff(hash_codes[1], sig_buff + 12); \ + int2buff(hash_codes[2], sig_buff + 16); \ + int2buff(hash_codes[3], sig_buff + 20); \ + } \ + else \ + { \ + memcpy(sig_buff + 8, hash_codes, 16); \ + } + +#define STORAGE_STAT_FILE_FAIL_LOG(result, client_ip, type_caption, filename) \ + if (result == ENOENT) \ + { \ + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, %s file: %s not exist", \ + __LINE__, client_ip, type_caption, filename); \ + } \ + else \ + { \ + logError("file: "__FILE__", line: %d, " \ + "call stat fail, client ip: %s, %s file: %s, "\ + "error no: %d, error info: %s", __LINE__, client_ip, \ + type_caption, filename, result, STRERROR(result)); \ + } + + +typedef struct +{ + char src_true_filename[128]; + char src_file_sig[64]; + int src_file_sig_len; +} SourceFileInfo; + +typedef struct +{ + SourceFileInfo src_file_info; + bool need_response; +} TrunkCreateLinkArg; + +static int storage_create_link_core(struct fast_task_info *pTask, \ + SourceFileInfo *pSourceFileInfo, \ + const char *src_filename, const char *master_filename, \ + const int master_filename_len, \ + const char *prefix_name, const char *file_ext_name, \ + char *filename, int *filename_len, const bool bNeedReponse); + +static int storage_set_link_file_meta(struct fast_task_info *pTask, \ + const SourceFileInfo *pSrcFileInfo, const char *link_filename); + +static FDFSStorageServer *get_storage_server(const char *storage_server_id) +{ + FDFSStorageServer targetServer; + FDFSStorageServer *pTargetServer; + FDFSStorageServer **ppFound; + + memset(&targetServer, 0, sizeof(targetServer)); + strcpy(targetServer.server.id, storage_server_id); + + pTargetServer = &targetServer; + ppFound = (FDFSStorageServer **)bsearch(&pTargetServer, \ + g_sorted_storages, g_storage_count, \ + sizeof(FDFSStorageServer *), storage_cmp_by_server_id); + if (ppFound == NULL) + { + return NULL; + } + else + { + return *ppFound; + } +} + +#define CHECK_AND_WRITE_TO_STAT_FILE1 \ + pthread_mutex_lock(&stat_count_thread_lock); \ +\ + if (pClientInfo->pSrcStorage == NULL) \ + { \ + pClientInfo->pSrcStorage = get_storage_server( \ + pClientInfo->storage_server_id); \ + } \ + if (pClientInfo->pSrcStorage != NULL) \ + { \ + pClientInfo->pSrcStorage->last_sync_src_timestamp = \ + pFileContext->timestamp2log; \ + g_sync_change_count++; \ + } \ +\ + g_storage_stat.last_sync_update = g_current_time; \ + ++g_stat_change_count; \ + pthread_mutex_unlock(&stat_count_thread_lock); + +#define CHECK_AND_WRITE_TO_STAT_FILE1_WITH_BYTES( \ + total_bytes, success_bytes, bytes) \ + pthread_mutex_lock(&stat_count_thread_lock); \ +\ + if (pClientInfo->pSrcStorage == NULL) \ + { \ + pClientInfo->pSrcStorage = get_storage_server( \ + pClientInfo->storage_server_id); \ + } \ + if (pClientInfo->pSrcStorage != NULL) \ + { \ + pClientInfo->pSrcStorage->last_sync_src_timestamp = \ + pFileContext->timestamp2log; \ + g_sync_change_count++; \ + } \ +\ + g_storage_stat.last_sync_update = g_current_time; \ + total_bytes += bytes; \ + success_bytes += bytes; \ + ++g_stat_change_count; \ + pthread_mutex_unlock(&stat_count_thread_lock); + +#define CHECK_AND_WRITE_TO_STAT_FILE2(total_count, success_count) \ + pthread_mutex_lock(&stat_count_thread_lock); \ + total_count++; \ + success_count++; \ + ++g_stat_change_count; \ + pthread_mutex_unlock(&stat_count_thread_lock); + +#define CHECK_AND_WRITE_TO_STAT_FILE2_WITH_BYTES(total_count, success_count, \ + total_bytes, success_bytes, bytes) \ + pthread_mutex_lock(&stat_count_thread_lock); \ + total_count++; \ + success_count++; \ + total_bytes += bytes; \ + success_bytes += bytes; \ + ++g_stat_change_count; \ + pthread_mutex_unlock(&stat_count_thread_lock); + +#define CHECK_AND_WRITE_TO_STAT_FILE3(total_count, success_count, timestamp) \ + pthread_mutex_lock(&stat_count_thread_lock); \ + total_count++; \ + success_count++; \ + timestamp = g_current_time; \ + ++g_stat_change_count; \ + pthread_mutex_unlock(&stat_count_thread_lock); + +#define CHECK_AND_WRITE_TO_STAT_FILE3_WITH_BYTES(total_count, success_count, \ + timestamp, total_bytes, success_bytes, bytes) \ + pthread_mutex_lock(&stat_count_thread_lock); \ + total_count++; \ + success_count++; \ + timestamp = g_current_time; \ + total_bytes += bytes; \ + success_bytes += bytes; \ + ++g_stat_change_count; \ + pthread_mutex_unlock(&stat_count_thread_lock); + +static void storage_log_access_log(struct fast_task_info *pTask, \ + const char *action, const int status) +{ + StorageClientInfo *pClientInfo; + struct timeval tv_end; + int time_used; + + pClientInfo = (StorageClientInfo *)pTask->arg; + gettimeofday(&tv_end, NULL); + time_used = (tv_end.tv_sec - pClientInfo->file_context. \ + tv_deal_start.tv_sec) * 1000 + + (tv_end.tv_usec - pClientInfo->file_context. \ + tv_deal_start.tv_usec) / 1000; + logAccess(&g_access_log_context, &(pClientInfo->file_context. \ + tv_deal_start), "%s %s %s %d %d "INT64_PRINTF_FORMAT" " \ + INT64_PRINTF_FORMAT, pTask->client_ip, action, \ + pClientInfo->file_context.fname2log, status, time_used, \ + pClientInfo->request_length, pClientInfo->total_length); +} + +#define STORAGE_ACCESS_STRCPY_FNAME2LOG(filename, filename_len, pClientInfo) \ + do \ + { \ + if (g_use_access_log) \ + { \ + if (filename_len < sizeof(pClientInfo-> \ + file_context.fname2log)) \ + { \ + memcpy(pClientInfo->file_context.fname2log, \ + filename, filename_len + 1); \ + } \ + else \ + { \ + memcpy(pClientInfo->file_context.fname2log, \ + filename, sizeof(pClientInfo-> \ + file_context.fname2log)); \ + *(pClientInfo->file_context.fname2log + \ + sizeof(pClientInfo->file_context. \ + fname2log) - 1) = '\0'; \ + } \ + } \ + } while (0) + + +#define STORAGE_ACCESS_LOG(pTask, action, status) \ + do \ + { \ + if (g_use_access_log && (status != STORAGE_STATUE_DEAL_FILE)) \ + { \ + storage_log_access_log(pTask, action, status); \ + } \ + } while (0) + + +static int storage_delete_file_auto(StorageFileContext *pFileContext) +{ + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + return trunk_file_delete(pFileContext->filename, + &(pFileContext->extra_info.upload.trunk_info)); + } + else + { + if (unlink(pFileContext->filename) == 0) + { + return 0; + } + else + { + return errno != 0 ? errno : ENOENT; + } + } +} + +static bool storage_is_slave_file(const char *remote_filename, \ + const int filename_len) +{ + int buff_len; + char buff[64]; + int64_t file_size; + + if (filename_len < FDFS_NORMAL_LOGIC_FILENAME_LENGTH) + { + logError("file: "__FILE__", line: %d, " \ + "filename is too short, length: %d < %d", \ + __LINE__, filename_len, FDFS_LOGIC_FILE_PATH_LEN \ + + FDFS_FILENAME_BASE64_LENGTH \ + + FDFS_FILE_EXT_NAME_MAX_LEN + 1); + return false; + } + + memset(buff, 0, sizeof(buff)); + base64_decode_auto(&g_fdfs_base64_context, (char *)remote_filename + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + buff, &buff_len); + + file_size = buff2long(buff + sizeof(int) * 2); + if (IS_TRUNK_FILE(file_size)) + { + return filename_len > FDFS_TRUNK_LOGIC_FILENAME_LENGTH; + } + + return filename_len > FDFS_NORMAL_LOGIC_FILENAME_LENGTH; +} + +static void storage_delete_file_log_error(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, delete file %s fail," \ + "errno: %d, error info: %s", __LINE__, \ + pTask->client_ip, pFileContext->filename, \ + err_no, STRERROR(err_no)); +} + +static void storage_sync_delete_file_log_error(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageFileContext *pFileContext; + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (err_no == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, " \ + "file %s not exist, " \ + "maybe delete later?", __LINE__, \ + STORAGE_PROTO_CMD_SYNC_DELETE_FILE, \ + pTask->client_ip, pFileContext->filename); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, delete file %s fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, err_no, STRERROR(err_no)); + } +} + +static void storage_sync_delete_file_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0 && pFileContext->sync_flag != '\0') + { + result = storage_binlog_write(pFileContext->timestamp2log, \ + pFileContext->sync_flag, pFileContext->fname2log); + } + else + { + result = err_no; + } + + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE1 + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); +} + +static void storage_sync_truncate_file_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0 && pFileContext->sync_flag != '\0') + { + set_file_utimes(pFileContext->filename, \ + pFileContext->timestamp2log); + result = storage_binlog_write(pFileContext->timestamp2log, \ + pFileContext->sync_flag, pFileContext->fname2log); + } + else + { + result = err_no; + } + + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE1 + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); +} + +static int storage_sync_copy_file_rename_filename( \ + StorageFileContext *pFileContext) +{ + char full_filename[MAX_PATH_SIZE + 256]; + char true_filename[128]; + int filename_len; + int result; + int store_path_index; + + filename_len = strlen(pFileContext->fname2log); + if ((result=storage_split_filename_ex(pFileContext->fname2log, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + if (rename(pFileContext->filename, full_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logWarning("file: "__FILE__", line: %d, " \ + "rename %s to %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pFileContext->filename, full_filename, \ + result, STRERROR(result)); + return result; + } + + return 0; +} + +static void storage_sync_copy_file_done_callback(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + result = err_no; + if (result == 0) + { + if (pFileContext->op == FDFS_STORAGE_FILE_OP_WRITE) + { + if (!(pFileContext->extra_info.upload.file_type & \ + _FILE_TYPE_TRUNK)) + { + set_file_utimes(pFileContext->filename, \ + pFileContext->timestamp2log); + + result = storage_sync_copy_file_rename_filename( \ + pFileContext); + } + + if (result == 0) + { + storage_binlog_write(pFileContext->timestamp2log, \ + pFileContext->sync_flag, pFileContext->fname2log); + } + } + else //FDFS_STORAGE_FILE_OP_DISCARD + { + storage_binlog_write(pFileContext->timestamp2log, \ + pFileContext->sync_flag, pFileContext->fname2log); + } + } + + if (pFileContext->op == FDFS_STORAGE_FILE_OP_WRITE) + { + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE1_WITH_BYTES( \ + g_storage_stat.total_sync_in_bytes, \ + g_storage_stat.success_sync_in_bytes, \ + pFileContext->end - pFileContext->start) + } + } + else //FDFS_STORAGE_FILE_OP_DISCARD + { + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE1 + } + + result = EEXIST; + } + if (result != 0) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_sync_in_bytes += \ + pClientInfo->total_offset; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); +} + +static void storage_sync_modify_file_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + result = err_no; + + if (pFileContext->op != FDFS_STORAGE_FILE_OP_DISCARD) + { + if (result == 0) + { + set_file_utimes(pFileContext->filename, \ + pFileContext->timestamp2log); + + storage_binlog_write(pFileContext->timestamp2log, \ + pFileContext->sync_flag, pFileContext->fname2log); + + CHECK_AND_WRITE_TO_STAT_FILE1_WITH_BYTES( \ + g_storage_stat.total_sync_in_bytes, \ + g_storage_stat.success_sync_in_bytes, \ + pFileContext->end - pFileContext->start) + } + } + else //FDFS_STORAGE_FILE_OP_DISCARD + { + if (result == 0) + { + struct stat file_stat; + + if (lstat(pFileContext->filename, &file_stat) != 0) + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, + pTask->client_ip, "regular", + pFileContext->filename) + } + else if (!S_ISREG(file_stat.st_mode)) + { + result = EEXIST; + } + else if (file_stat.st_size < pFileContext->end) + { + result = ENOENT; //need to resync + } + else + { + result = EEXIST; + } + + CHECK_AND_WRITE_TO_STAT_FILE1 + } + } + + if (result != 0) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_sync_in_bytes += \ + pClientInfo->total_offset; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); +} + +#define STORAGE_NIO_NOTIFY_CLOSE(pTask) \ +do { \ + ((StorageClientInfo *)pTask->arg)->stage = FDFS_STORAGE_STAGE_NIO_CLOSE; \ + storage_nio_notify(pTask); \ + } while (0) + +static void storage_get_metadata_done_callback(struct fast_task_info *pTask, \ + const int err_no) +{ + TrackerHeader *pHeader; + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_GET_METADATA, \ + err_no); + + if (err_no != 0) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_get_meta_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + + if (pTask->length == sizeof(TrackerHeader)) //never response + { + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = err_no; + storage_nio_notify(pTask); + } + else + { + STORAGE_NIO_NOTIFY_CLOSE(pTask); + } + } + else + { + CHECK_AND_WRITE_TO_STAT_FILE2( \ + g_storage_stat.total_get_meta_count, \ + g_storage_stat.success_get_meta_count) + + storage_nio_notify(pTask); + } +} + +static void storage_download_file_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_DOWNLOAD_FILE, \ + err_no); + + pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context); + if (err_no != 0) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_download_count++; + g_storage_stat.total_download_bytes += \ + pFileContext->offset - pFileContext->start; + pthread_mutex_unlock(&stat_count_thread_lock); + + if (pTask->length == sizeof(TrackerHeader)) //never response + { + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = err_no; + storage_nio_notify(pTask); + } + else + { + STORAGE_NIO_NOTIFY_CLOSE(pTask); + } + } + else + { + CHECK_AND_WRITE_TO_STAT_FILE2_WITH_BYTES( \ + g_storage_stat.total_download_count, \ + g_storage_stat.success_download_count, \ + g_storage_stat.total_download_bytes, \ + g_storage_stat.success_download_bytes, \ + pFileContext->end - pFileContext->start) + + storage_nio_notify(pTask); + } +} + +static int storage_do_delete_meta_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + GroupArray *pGroupArray; + char meta_filename[MAX_PATH_SIZE + 256]; + char true_filename[128]; + char value[128]; + FDHTKeyInfo key_info_fid; + FDHTKeyInfo key_info_ref; + FDHTKeyInfo key_info_sig; + char *pValue; + int value_len; + int src_file_nlink; + int result; + int store_path_index; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (pFileContext->extra_info.upload.file_type & \ + _FILE_TYPE_TRUNK) + { + int filename_len = strlen(pFileContext->fname2log); + if ((result=storage_split_filename_ex(pFileContext->fname2log, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + + sprintf(meta_filename, "%s/data/%s"FDFS_STORAGE_META_FILE_EXT, \ + g_fdfs_store_paths.paths[store_path_index], true_filename); + } + else + { + sprintf(meta_filename, "%s"FDFS_STORAGE_META_FILE_EXT, \ + pFileContext->filename); + } + if (fileExists(meta_filename)) + { + if (unlink(meta_filename) != 0) + { + if (errno != ENOENT) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, delete file %s fail," \ + "errno: %d, error info: %s", __LINE__,\ + pTask->client_ip, meta_filename, \ + result, STRERROR(result)); + return result; + } + } + else + { + sprintf(meta_filename, "%s"FDFS_STORAGE_META_FILE_EXT, \ + pFileContext->fname2log); + result = storage_binlog_write(g_current_time, \ + STORAGE_OP_TYPE_SOURCE_DELETE_FILE, meta_filename); + if (result != 0) + { + return result; + } + } + } + + src_file_nlink = -1; + if (g_check_file_duplicate) + { + pGroupArray=&((g_nio_thread_data+pClientInfo->nio_thread_index)\ + ->group_array); + memset(&key_info_sig, 0, sizeof(key_info_sig)); + key_info_sig.namespace_len = g_namespace_len; + memcpy(key_info_sig.szNameSpace, g_key_namespace, \ + g_namespace_len); + key_info_sig.obj_id_len = snprintf(\ + key_info_sig.szObjectId, \ + sizeof(key_info_sig.szObjectId), "%s/%s", \ + g_group_name, pFileContext->fname2log); + + key_info_sig.key_len = sizeof(FDHT_KEY_NAME_FILE_SIG)-1; + memcpy(key_info_sig.szKey, FDHT_KEY_NAME_FILE_SIG, \ + key_info_sig.key_len); + pValue = value; + value_len = sizeof(value) - 1; + result = fdht_get_ex1(pGroupArray, g_keep_alive, \ + &key_info_sig, FDHT_EXPIRES_NONE, \ + &pValue, &value_len, malloc); + if (result == 0) + { + memcpy(&key_info_fid, &key_info_sig, \ + sizeof(FDHTKeyInfo)); + key_info_fid.obj_id_len = value_len; + memcpy(key_info_fid.szObjectId, pValue, \ + value_len); + + key_info_fid.key_len = sizeof(FDHT_KEY_NAME_FILE_ID) - 1; + memcpy(key_info_fid.szKey, FDHT_KEY_NAME_FILE_ID, \ + key_info_fid.key_len); + value_len = sizeof(value) - 1; + result = fdht_get_ex1(pGroupArray, \ + g_keep_alive, &key_info_fid, \ + FDHT_EXPIRES_NONE, &pValue, \ + &value_len, malloc); + if (result == 0) + { + memcpy(&key_info_ref, &key_info_sig, \ + sizeof(FDHTKeyInfo)); + key_info_ref.obj_id_len = value_len; + memcpy(key_info_ref.szObjectId, pValue, + value_len); + key_info_ref.key_len = \ + sizeof(FDHT_KEY_NAME_REF_COUNT)-1; + memcpy(key_info_ref.szKey, \ + FDHT_KEY_NAME_REF_COUNT, \ + key_info_ref.key_len); + value_len = sizeof(value) - 1; + + result = fdht_get_ex1(pGroupArray, \ + g_keep_alive, &key_info_ref, \ + FDHT_EXPIRES_NONE, &pValue, \ + &value_len, malloc); + if (result == 0) + { + *(pValue + value_len) = '\0'; + src_file_nlink = atoi(pValue); + } + else if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_get fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + return result; + } + } + else if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_get fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + return result; + } + } + else if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_get fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + return result; + } + } + + if (src_file_nlink < 0) + { + return 0; + } + + if (g_check_file_duplicate) + { + char *pSeperator; + struct stat stat_buf; + FDFSTrunkHeader trunkHeader; + + pGroupArray=&((g_nio_thread_data+pClientInfo->nio_thread_index)\ + ->group_array); + if ((result=fdht_delete_ex(pGroupArray, g_keep_alive, \ + &key_info_sig)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_delete fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + } + + value_len = sizeof(value) - 1; + result = fdht_inc_ex(pGroupArray, g_keep_alive, \ + &key_info_ref, FDHT_EXPIRES_NEVER, -1, \ + value, &value_len); + if (result != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_inc fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + return result; + } + + if (!(value_len == 1 && *value == '0')) //value == 0 + { + return 0; + } + + if ((result=fdht_delete_ex(pGroupArray, g_keep_alive, \ + &key_info_fid)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_delete fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + } + if ((result=fdht_delete_ex(pGroupArray, g_keep_alive, \ + &key_info_ref)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_delete fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + } + + *(key_info_ref.szObjectId+key_info_ref.obj_id_len)='\0'; + pSeperator = strchr(key_info_ref.szObjectId, '/'); + if (pSeperator == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid file_id: %s", __LINE__, \ + key_info_ref.szObjectId); + return 0; + } + + pSeperator++; + value_len = key_info_ref.obj_id_len - (pSeperator - \ + key_info_ref.szObjectId); + memcpy(value, pSeperator, value_len + 1); + if ((result=storage_split_filename_ex(value, &value_len, \ + true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, \ + value_len)) != 0) + { + return result; + } + + if ((result=trunk_file_lstat(store_path_index, true_filename, \ + value_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &trunkHeader)) != 0) + { + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "logic", value) + return 0; + } + + if (IS_TRUNK_FILE_BY_ID(pFileContext->extra_info. \ + upload.trunk_info)) + { + trunk_get_full_filename(&(pFileContext->extra_info. \ + upload.trunk_info), pFileContext->filename, \ + sizeof(pFileContext->filename)); + } + else + { + sprintf(pFileContext->filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + } + + if ((result=storage_delete_file_auto(pFileContext)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, delete logic source file " \ + "%s fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + value, errno, STRERROR(errno)); + return 0; + } + + storage_binlog_write(g_current_time, \ + STORAGE_OP_TYPE_SOURCE_DELETE_FILE, value); + pFileContext->delete_flag |= STORAGE_DELETE_FLAG_FILE; + } + + return 0; +} + +static void storage_delete_fdfs_file_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0) + { + if (pFileContext->extra_info.upload.file_type & \ + _FILE_TYPE_TRUNK) + { + trunk_client_trunk_free_space( \ + &(pFileContext->extra_info.upload.trunk_info)); + } + + result = storage_binlog_write(g_current_time, \ + STORAGE_OP_TYPE_SOURCE_DELETE_FILE, \ + pFileContext->fname2log); + } + else + { + result = err_no; + } + + if (result == 0) + { + result = storage_do_delete_meta_file(pTask); + } + + if (result != 0) + { + pthread_mutex_lock(&stat_count_thread_lock); + if (pFileContext->delete_flag == STORAGE_DELETE_FLAG_NONE ||\ + (pFileContext->delete_flag & STORAGE_DELETE_FLAG_FILE)) + { + g_storage_stat.total_delete_count++; + } + if (pFileContext->delete_flag & STORAGE_DELETE_FLAG_LINK) + { + g_storage_stat.total_delete_link_count++; + } + pthread_mutex_unlock(&stat_count_thread_lock); + } + else + { + if (pFileContext->delete_flag & STORAGE_DELETE_FLAG_FILE) + { + CHECK_AND_WRITE_TO_STAT_FILE3( \ + g_storage_stat.total_delete_count, \ + g_storage_stat.success_delete_count, \ + g_storage_stat.last_source_update) + } + + if (pFileContext->delete_flag & STORAGE_DELETE_FLAG_LINK) + { + CHECK_AND_WRITE_TO_STAT_FILE3( \ + g_storage_stat.total_delete_link_count, \ + g_storage_stat.success_delete_link_count, \ + g_storage_stat.last_source_update) + } + + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_DELETE_FILE, result); + + storage_nio_notify(pTask); +} + +static void storage_upload_file_done_callback(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + result = trunk_client_trunk_alloc_confirm( \ + &(pFileContext->extra_info.upload.trunk_info), err_no); + if (err_no != 0) + { + result = err_no; + } + } + else + { + result = err_no; + } + + if (result == 0) + { + result = storage_service_upload_file_done(pTask); + if (result == 0) + { + if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE) + { + result = storage_binlog_write(\ + pFileContext->timestamp2log, \ + STORAGE_OP_TYPE_SOURCE_CREATE_FILE, \ + pFileContext->fname2log); + } + } + } + + if (result == 0) + { + int filename_len; + char *p; + + if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE) + { + CHECK_AND_WRITE_TO_STAT_FILE3_WITH_BYTES( \ + g_storage_stat.total_upload_count, \ + g_storage_stat.success_upload_count, \ + g_storage_stat.last_source_update, \ + g_storage_stat.total_upload_bytes, \ + g_storage_stat.success_upload_bytes, \ + pFileContext->end - pFileContext->start) + } + + filename_len = strlen(pFileContext->fname2log); + pClientInfo->total_length = sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len; + p = pTask->data + sizeof(TrackerHeader); + memcpy(p, pFileContext->extra_info.upload.group_name, \ + FDFS_GROUP_NAME_MAX_LEN); + p += FDFS_GROUP_NAME_MAX_LEN; + memcpy(p, pFileContext->fname2log, filename_len); + } + else + { + pthread_mutex_lock(&stat_count_thread_lock); + if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE) + { + g_storage_stat.total_upload_count++; + g_storage_stat.total_upload_bytes += \ + pClientInfo->total_offset; + } + pthread_mutex_unlock(&stat_count_thread_lock); + + pClientInfo->total_length = sizeof(TrackerHeader); + } + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_UPLOAD_FILE, result); + + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); +} + +static void storage_trunk_create_link_file_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + TrunkCreateLinkArg *pCreateLinkArg; + SourceFileInfo *pSourceFileInfo; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + pCreateLinkArg = (TrunkCreateLinkArg *)pClientInfo->extra_arg; + pSourceFileInfo = &(pCreateLinkArg->src_file_info); + + result = trunk_client_trunk_alloc_confirm( \ + &(pFileContext->extra_info.upload.trunk_info), err_no); + if (err_no != 0) + { + result = err_no; + } + + if (result == 0) + { + result = storage_service_upload_file_done(pTask); + if (result == 0) + { + char src_filename[128]; + char binlog_msg[256]; + + sprintf(src_filename, \ + "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + pFileContext->extra_info.upload.trunk_info. \ + path.store_path_index, \ + pSourceFileInfo->src_true_filename); + + sprintf(binlog_msg, "%s %s", \ + pFileContext->fname2log, src_filename); + result = storage_binlog_write( \ + pFileContext->timestamp2log, \ + STORAGE_OP_TYPE_SOURCE_CREATE_LINK, \ + binlog_msg); + } + } + + if (result == 0) + { + int filename_len; + char *p; + + CHECK_AND_WRITE_TO_STAT_FILE3( \ + g_storage_stat.total_create_link_count, \ + g_storage_stat.success_create_link_count, \ + g_storage_stat.last_source_update) + + filename_len = strlen(pFileContext->fname2log); + pClientInfo->total_length = sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len; + p = pTask->data + sizeof(TrackerHeader); + memcpy(p, pFileContext->extra_info.upload.group_name, \ + FDFS_GROUP_NAME_MAX_LEN); + p += FDFS_GROUP_NAME_MAX_LEN; + memcpy(p, pFileContext->fname2log, filename_len); + } + else + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_create_link_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + pClientInfo->total_length = sizeof(TrackerHeader); + } + + storage_set_link_file_meta(pTask, pSourceFileInfo, \ + pFileContext->fname2log); + + if (pCreateLinkArg->need_response) + { + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); + } +} + +static void storage_append_file_done_callback(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + char extra[64]; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0) + { + struct stat stat_buf; + if (stat(pFileContext->filename, &stat_buf) == 0) + { + pFileContext->timestamp2log = stat_buf.st_mtime; + } + else + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "regular", pFileContext->filename) + } + + sprintf(extra, INT64_PRINTF_FORMAT" "INT64_PRINTF_FORMAT, \ + pFileContext->start, \ + pFileContext->end - pFileContext->start); + result = storage_binlog_write_ex(pFileContext->timestamp2log, \ + pFileContext->sync_flag, \ + pFileContext->fname2log, extra); + } + else + { + result = err_no; + } + + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE3_WITH_BYTES( \ + g_storage_stat.total_append_count, \ + g_storage_stat.success_append_count, \ + g_storage_stat.last_source_update, \ + g_storage_stat.total_append_bytes, \ + g_storage_stat.success_append_bytes, \ + pFileContext->end - pFileContext->start) + } + else + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_append_count++; + g_storage_stat.total_append_bytes += pClientInfo->total_offset; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(0, pHeader->pkg_len); + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_APPEND_FILE, result); + + storage_nio_notify(pTask); +} + +static void storage_modify_file_done_callback(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + char extra[64]; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0) + { + struct stat stat_buf; + if (stat(pFileContext->filename, &stat_buf) == 0) + { + pFileContext->timestamp2log = stat_buf.st_mtime; + } + else + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "regular", pFileContext->filename) + } + + sprintf(extra, INT64_PRINTF_FORMAT" "INT64_PRINTF_FORMAT, \ + pFileContext->start, \ + pFileContext->end - pFileContext->start); + result = storage_binlog_write_ex(pFileContext->timestamp2log, \ + pFileContext->sync_flag, \ + pFileContext->fname2log, extra); + } + else + { + result = err_no; + } + + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE3_WITH_BYTES( \ + g_storage_stat.total_modify_count, \ + g_storage_stat.success_modify_count, \ + g_storage_stat.last_source_update, \ + g_storage_stat.total_modify_bytes, \ + g_storage_stat.success_modify_bytes, \ + pFileContext->end - pFileContext->start) + } + else + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_modify_count++; + g_storage_stat.total_modify_bytes += pClientInfo->total_offset; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(0, pHeader->pkg_len); + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_MODIFY_FILE, result); + + storage_nio_notify(pTask); +} + +static void storage_do_truncate_file_done_callback(struct fast_task_info *pTask, \ + const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + char extra[64]; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0) + { + struct stat stat_buf; + if (stat(pFileContext->filename, &stat_buf) == 0) + { + pFileContext->timestamp2log = stat_buf.st_mtime; + } + else + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "regular", pFileContext->filename) + } + sprintf(extra, INT64_PRINTF_FORMAT" "INT64_PRINTF_FORMAT, \ + pFileContext->end - pFileContext->start, + pFileContext->offset); + result = storage_binlog_write_ex(pFileContext->timestamp2log, \ + pFileContext->sync_flag, \ + pFileContext->fname2log, extra); + } + else + { + result = err_no; + } + + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE3( \ + g_storage_stat.total_truncate_count, \ + g_storage_stat.success_truncate_count, \ + g_storage_stat.last_source_update) + } + else + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_truncate_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(0, pHeader->pkg_len); + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_TRUNCATE_FILE, result); + + storage_nio_notify(pTask); +} + +static void storage_set_metadata_done_callback( \ + struct fast_task_info *pTask, const int err_no) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if (err_no == 0) + { + if (pFileContext->sync_flag != '\0') + { + result = storage_binlog_write(pFileContext->timestamp2log, \ + pFileContext->sync_flag, pFileContext->fname2log); + } + else + { + result = err_no; + } + } + else + { + result = err_no; + } + + if (result != 0) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_set_meta_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + } + else + { + CHECK_AND_WRITE_TO_STAT_FILE3( \ + g_storage_stat.total_set_meta_count, \ + g_storage_stat.success_set_meta_count, \ + g_storage_stat.last_source_update) + } + + pClientInfo->total_length = sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_SET_METADATA, result); + + storage_nio_notify(pTask); +} + +int storage_service_init() +{ + int result; + int bytes; + struct storage_nio_thread_data *pThreadData; + struct storage_nio_thread_data *pDataEnd; + pthread_t tid; + pthread_attr_t thread_attr; + + if ((result=init_pthread_lock(&g_storage_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&path_index_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&stat_count_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_attr(&thread_attr, g_thread_stack_size)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_attr fail, program exit!", __LINE__); + return result; + } + + if ((result=free_queue_init(g_max_connections, g_buff_size, \ + g_buff_size, sizeof(StorageClientInfo))) != 0) + { + return result; + } + + bytes = sizeof(struct storage_nio_thread_data) * g_work_threads; + g_nio_thread_data = (struct storage_nio_thread_data *)malloc(bytes); + if (g_nio_thread_data == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(g_nio_thread_data, 0, bytes); + + g_storage_thread_count = 0; + pDataEnd = g_nio_thread_data + g_work_threads; + for (pThreadData=g_nio_thread_data; pThreadDatathread_data.ev_puller, + g_max_connections + 2, 1000, 0) != 0) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "ioevent_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + result = fast_timer_init(&pThreadData->thread_data.timer, + 2 * g_fdfs_network_timeout, g_current_time); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "fast_timer_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if (pipe(pThreadData->thread_data.pipe_fds) != 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "call pipe fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + break; + } + +#if defined(OS_LINUX) + if ((result=fd_add_flags(pThreadData->thread_data.pipe_fds[0], \ + O_NONBLOCK | O_NOATIME)) != 0) + { + break; + } +#else + if ((result=fd_add_flags(pThreadData->thread_data.pipe_fds[0], \ + O_NONBLOCK)) != 0) + { + break; + } +#endif + + if ((result=pthread_create(&tid, &thread_attr, \ + work_thread_entrance, pThreadData)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "create thread failed, startup threads: %d, " \ + "errno: %d, error info: %s", \ + __LINE__, g_storage_thread_count, \ + result, STRERROR(result)); + break; + } + else + { + if ((result=pthread_mutex_lock(&g_storage_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + g_storage_thread_count++; + if ((result=pthread_mutex_unlock(&g_storage_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + } + } + + pthread_attr_destroy(&thread_attr); + + last_stat_change_count = g_stat_change_count; + + //DO NOT support direct IO !!! + //g_extra_open_file_flags = g_disk_rw_direct ? O_DIRECT : 0; + + if (result != 0) + { + return result; + } + + return result; +} + +void storage_service_destroy() +{ + pthread_mutex_destroy(&g_storage_thread_lock); + pthread_mutex_destroy(&path_index_thread_lock); + pthread_mutex_destroy(&stat_count_thread_lock); +} + +int storage_terminate_threads() +{ + struct storage_nio_thread_data *pThreadData; + struct storage_nio_thread_data *pDataEnd; + struct fast_task_info *pTask; + StorageClientInfo *pClientInfo; + long task_addr; + int quit_sock; + + if (g_nio_thread_data != NULL) + { + pDataEnd = g_nio_thread_data + g_work_threads; + quit_sock = 0; + + for (pThreadData=g_nio_thread_data; pThreadDataarg; + pTask->event.fd = quit_sock; + pClientInfo->nio_thread_index = pThreadData - g_nio_thread_data; + + task_addr = (long)pTask; + if (write(pThreadData->thread_data.pipe_fds[1], &task_addr, \ + sizeof(task_addr)) != sizeof(task_addr)) + { + logError("file: "__FILE__", line: %d, " \ + "call write failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + } + } + + return 0; +} + +static void *accept_thread_entrance(void* arg) +{ + int server_sock; + int incomesock; + struct sockaddr_in inaddr; + socklen_t sockaddr_len; + in_addr_t client_addr; + char szClientIp[IP_ADDRESS_SIZE]; + long task_addr; + struct fast_task_info *pTask; + StorageClientInfo *pClientInfo; + struct storage_nio_thread_data *pThreadData; + + server_sock = (long)arg; + while (g_continue_flag) + { + sockaddr_len = sizeof(inaddr); + incomesock = accept(server_sock, (struct sockaddr*)&inaddr, \ + &sockaddr_len); + if (incomesock < 0) //error + { + if (!(errno == EINTR || errno == EAGAIN)) + { + logError("file: "__FILE__", line: %d, " \ + "accept failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + + continue; + } + + client_addr = getPeerIpaddr(incomesock, \ + szClientIp, IP_ADDRESS_SIZE); + if (g_allow_ip_count >= 0) + { + if (bsearch(&client_addr, g_allow_ip_addrs, \ + g_allow_ip_count, sizeof(in_addr_t), \ + cmp_by_ip_addr_t) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "ip addr %s is not allowed to access", \ + __LINE__, szClientIp); + + close(incomesock); + continue; + } + } + + if (tcpsetnonblockopt(incomesock) != 0) + { + close(incomesock); + continue; + } + + pTask = free_queue_pop(); + if (pTask == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc task buff failed", \ + __LINE__); + close(incomesock); + continue; + } + + pClientInfo = (StorageClientInfo *)pTask->arg; + pTask->event.fd = incomesock; + pClientInfo->stage = FDFS_STORAGE_STAGE_NIO_INIT; + pClientInfo->nio_thread_index = pTask->event.fd % g_work_threads; + pThreadData = g_nio_thread_data + pClientInfo->nio_thread_index; + + strcpy(pTask->client_ip, szClientIp); + + task_addr = (long)pTask; + if (write(pThreadData->thread_data.pipe_fds[1], &task_addr, \ + sizeof(task_addr)) != sizeof(task_addr)) + { + close(incomesock); + free_queue_push(pTask); + logError("file: "__FILE__", line: %d, " \ + "call write failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + } + + return NULL; +} + +void storage_accept_loop(int server_sock) +{ + if (g_accept_threads > 1) + { + pthread_t tid; + pthread_attr_t thread_attr; + int result; + int i; + + if ((result=init_pthread_attr(&thread_attr, g_thread_stack_size)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "init_pthread_attr fail!", __LINE__); + } + else + { + for (i=1; iarg; + pThreadData = g_nio_thread_data + pClientInfo->nio_thread_index; + + task_addr = (long)pTask; + if (write(pThreadData->thread_data.pipe_fds[1], &task_addr, \ + sizeof(task_addr)) != sizeof(task_addr)) + { + int result; + result = errno != 0 ? errno : EIO; + logCrit("file: "__FILE__", line: %d, " \ + "call write failed, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + abort(); + } +} + +static void *work_thread_entrance(void* arg) +{ + int result; + struct storage_nio_thread_data *pThreadData; + + pThreadData = (struct storage_nio_thread_data *)arg; + if (g_check_file_duplicate) + { + if ((result=fdht_copy_group_array(&(pThreadData->group_array),\ + &g_group_array)) != 0) + { + pthread_mutex_lock(&g_storage_thread_lock); + g_storage_thread_count--; + pthread_mutex_unlock(&g_storage_thread_lock); + return NULL; + } + } + + ioevent_loop(&pThreadData->thread_data, storage_recv_notify_read, + task_finish_clean_up, &g_continue_flag); + ioevent_destroy(&pThreadData->thread_data.ev_puller); + + if (g_check_file_duplicate) + { + if (g_keep_alive) + { + fdht_disconnect_all_servers(&(pThreadData->group_array)); + } + + fdht_free_group_array(&(pThreadData->group_array)); + } + + if ((result=pthread_mutex_lock(&g_storage_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + g_storage_thread_count--; + if ((result=pthread_mutex_unlock(&g_storage_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + logDebug("file: "__FILE__", line: %d, " \ + "nio thread exited, thread count: %d", \ + __LINE__, g_storage_thread_count); + + return NULL; +} + +int storage_get_storage_path_index(int *store_path_index) +{ + int i; + + *store_path_index = g_store_path_index; + if (g_store_path_mode == FDFS_STORE_PATH_LOAD_BALANCE) + { + if (*store_path_index < 0 || *store_path_index >= \ + g_fdfs_store_paths.count) + { + return ENOSPC; + } + } + else + { + if (*store_path_index >= g_fdfs_store_paths.count) + { + *store_path_index = 0; + } + + if (!storage_check_reserved_space_path(g_path_space_list \ + [*store_path_index].total_mb, g_path_space_list \ + [*store_path_index].free_mb, g_avg_storage_reserved_mb)) + { + for (i=0; i= g_fdfs_store_paths.count) + { + g_store_path_index = 0; + } + } + + return 0; +} + +void storage_get_store_path(const char *filename, const int filename_len, \ + int *sub_path_high, int *sub_path_low) +{ + int n; + int result; + + if (g_file_distribute_path_mode == FDFS_FILE_DIST_PATH_ROUND_ROBIN) + { + *sub_path_high = g_dist_path_index_high; + *sub_path_low = g_dist_path_index_low; + + if (++g_dist_write_file_count >= g_file_distribute_rotate_count) + { + g_dist_write_file_count = 0; + + if ((result=pthread_mutex_lock( \ + &path_index_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + ++g_dist_path_index_low; + if (g_dist_path_index_low >= g_subdir_count_per_path) + { //rotate + g_dist_path_index_high++; + if (g_dist_path_index_high >= \ + g_subdir_count_per_path) //rotate + { + g_dist_path_index_high = 0; + } + g_dist_path_index_low = 0; + } + + ++g_stat_change_count; + + if ((result=pthread_mutex_unlock( \ + &path_index_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + } + } //random + else + { + n = PJWHash(filename, filename_len) % (1 << 16); + *sub_path_high = ((n >> 8) & 0xFF) % g_subdir_count_per_path; + *sub_path_low = (n & 0xFF) % g_subdir_count_per_path; + } +} + +#define COMBINE_RAND_FILE_SIZE(file_size, masked_file_size) \ + do \ + { \ + int r; \ + r = (rand() & 0x007FFFFF) | 0x80000000; \ + masked_file_size = (((int64_t)r) << 32 ) | file_size; \ + } while (0) + + +static int storage_gen_filename(StorageClientInfo *pClientInfo, \ + const int64_t file_size, const int crc32, \ + const char *szFormattedExt, const int ext_name_len, \ + const time_t timestamp, char *filename, int *filename_len) +{ + char buff[sizeof(int) * 5]; + char encoded[sizeof(int) * 8 + 1]; + int len; + int64_t masked_file_size; + FDFSTrunkFullInfo *pTrunkInfo; + + pTrunkInfo = &(pClientInfo->file_context.extra_info.upload.trunk_info); + int2buff(htonl(g_server_id_in_filename), buff); + int2buff(timestamp, buff+sizeof(int)); + if ((file_size >> 32) != 0) + { + masked_file_size = file_size; + } + else + { + COMBINE_RAND_FILE_SIZE(file_size, masked_file_size); + } + long2buff(masked_file_size, buff+sizeof(int)*2); + int2buff(crc32, buff+sizeof(int)*4); + + base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(int) * 5, encoded, \ + filename_len, false); + + if (!pClientInfo->file_context.extra_info.upload.if_sub_path_alloced) + { + int sub_path_high; + int sub_path_low; + storage_get_store_path(encoded, *filename_len, \ + &sub_path_high, &sub_path_low); + + pTrunkInfo->path.sub_path_high = sub_path_high; + pTrunkInfo->path.sub_path_low = sub_path_low; + + pClientInfo->file_context.extra_info.upload. \ + if_sub_path_alloced = true; + } + + len = sprintf(filename, FDFS_STORAGE_DATA_DIR_FORMAT"/" \ + FDFS_STORAGE_DATA_DIR_FORMAT"/", \ + pTrunkInfo->path.sub_path_high, + pTrunkInfo->path.sub_path_low); + memcpy(filename+len, encoded, *filename_len); + memcpy(filename+len+(*filename_len), szFormattedExt, ext_name_len); + *filename_len += len + ext_name_len; + *(filename + (*filename_len)) = '\0'; + + return 0; +} + +static int storage_sort_metadata_buff(char *meta_buff, const int meta_size) +{ + FDFSMetaData *meta_list; + int meta_count; + int meta_bytes; + int result; + + meta_list = fdfs_split_metadata(meta_buff, &meta_count, &result); + if (meta_list == NULL) + { + return result; + } + + qsort((void *)meta_list, meta_count, sizeof(FDFSMetaData), \ + metadata_cmp_by_name); + + fdfs_pack_metadata(meta_list, meta_count, meta_buff, &meta_bytes); + free(meta_list); + + return 0; +} + +static void storage_format_ext_name(const char *file_ext_name, \ + char *szFormattedExt) +{ + int i; + int ext_name_len; + int pad_len; + char *p; + + ext_name_len = strlen(file_ext_name); + if (ext_name_len == 0) + { + pad_len = FDFS_FILE_EXT_NAME_MAX_LEN + 1; + } + else + { + pad_len = FDFS_FILE_EXT_NAME_MAX_LEN - ext_name_len; + } + + p = szFormattedExt; + for (i=0; i 0) + { + *p++ = '.'; + memcpy(p, file_ext_name, ext_name_len); + p += ext_name_len; + } + *p = '\0'; +} + +static int storage_get_filename(StorageClientInfo *pClientInfo, \ + const int start_time, const int64_t file_size, const int crc32, \ + const char *szFormattedExt, char *filename, \ + int *filename_len, char *full_filename) +{ + int i; + int result; + int store_path_index; + + store_path_index = pClientInfo->file_context.extra_info.upload. + trunk_info.path.store_path_index; + for (i=0; i<10; i++) + { + if ((result=storage_gen_filename(pClientInfo, file_size, \ + crc32, szFormattedExt, FDFS_FILE_EXT_NAME_MAX_LEN+1, \ + start_time, filename, filename_len)) != 0) + { + return result; + } + + sprintf(full_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], filename); + if (!fileExists(full_filename)) + { + break; + } + + *full_filename = '\0'; + } + + if (*full_filename == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "Can't generate uniq filename", __LINE__); + *filename = '\0'; + *filename_len = 0; + return ENOENT; + } + + return 0; +} + +static int storage_client_create_link_wrapper(struct fast_task_info *pTask, \ + const char *master_filename, \ + const char *src_filename, const int src_filename_len, \ + const char *src_file_sig, const int src_file_sig_len, \ + const char *group_name, const char *prefix_name, \ + const char *file_ext_name, \ + char *remote_filename, int *filename_len) +{ + int result; + int src_store_path_index; + ConnectionInfo trackerServer; + ConnectionInfo *pTracker; + ConnectionInfo storageServer; + ConnectionInfo *pStorageServer; + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + SourceFileInfo sourceFileInfo; + bool bCreateDirectly; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if ((pTracker=tracker_get_connection_r(&trackerServer, &result)) == NULL) + { + return result; + } + + if (strcmp(group_name, g_group_name) != 0) + { + pStorageServer = NULL; + bCreateDirectly = false; + } + else + { + result = tracker_query_storage_update(pTracker, \ + &storageServer, group_name, src_filename); + if (result != 0) + { + tracker_disconnect_server_ex(pTracker, true); + return result; + } + + if (is_local_host_ip(storageServer.ip_addr)) + { + bCreateDirectly = true; + } + else + { + bCreateDirectly = false; + } + + if (!bCreateDirectly) + { + if ((pStorageServer=tracker_connect_server( \ + &storageServer, &result)) == NULL) + { + tracker_disconnect_server(pTracker); + return result; + } + } + else + { + pStorageServer = NULL; + } + } + + if (bCreateDirectly) + { + sourceFileInfo.src_file_sig_len = src_file_sig_len; + memcpy(sourceFileInfo.src_file_sig, src_file_sig, \ + src_file_sig_len); + *(sourceFileInfo.src_file_sig + src_file_sig_len) = '\0'; + + *filename_len = src_filename_len; + if ((result=storage_split_filename_ex(src_filename, \ + filename_len, sourceFileInfo.src_true_filename, \ + &src_store_path_index)) != 0) + { + tracker_disconnect_server(pTracker); + return result; + } + + pFileContext->extra_info.upload.trunk_info.path. \ + store_path_index = src_store_path_index; + result = storage_create_link_core(pTask, \ + &sourceFileInfo, src_filename, \ + master_filename, strlen(master_filename), \ + prefix_name, file_ext_name, \ + remote_filename, filename_len, false); + if (result == STORAGE_STATUE_DEAL_FILE) + { + result = 0; + } + } + else + { + result = storage_client_create_link(pTracker, \ + pStorageServer, master_filename, \ + src_filename, src_filename_len, \ + src_file_sig, src_file_sig_len, \ + group_name, prefix_name, \ + file_ext_name, remote_filename, filename_len); + if (pStorageServer != NULL) + { + tracker_disconnect_server_ex(pStorageServer, result != 0); + } + } + + tracker_disconnect_server(pTracker); + + return result; +} + +static int storage_service_upload_file_done(struct fast_task_info *pTask) +{ + int result; + int filename_len; + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int64_t file_size; + int64_t file_size_in_name; + time_t end_time; + char new_fname2log[128]; + char new_full_filename[MAX_PATH_SIZE+64]; + char new_filename[128]; + int new_filename_len; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + file_size = pFileContext->end - pFileContext->start; + + *new_full_filename = '\0'; + *new_filename = '\0'; + new_filename_len = 0; + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + end_time = pFileContext->extra_info.upload.start_time; + COMBINE_RAND_FILE_SIZE(file_size, file_size_in_name); + file_size_in_name |= FDFS_TRUNK_FILE_MARK_SIZE; + } + else + { + struct stat stat_buf; + if (stat(pFileContext->filename, &stat_buf) == 0) + { + end_time = stat_buf.st_mtime; + } + else + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "regular", pFileContext->filename) + end_time = g_current_time; + } + + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_APPENDER) + { + COMBINE_RAND_FILE_SIZE(0, file_size_in_name); + file_size_in_name |= FDFS_APPENDER_FILE_SIZE; + } + else + { + file_size_in_name = file_size; + } + } + + if ((result=storage_get_filename(pClientInfo, end_time, \ + file_size_in_name, pFileContext->crc32, \ + pFileContext->extra_info.upload.formatted_ext_name, \ + new_filename, &new_filename_len, new_full_filename)) != 0) + { + storage_delete_file_auto(pFileContext); + return result; + } + + memcpy(pFileContext->extra_info.upload.group_name, g_group_name, \ + FDFS_GROUP_NAME_MAX_LEN + 1); + sprintf(new_fname2log, "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + pFileContext->extra_info.upload.trunk_info.path. \ + store_path_index, new_filename); + + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + char trunk_buff[FDFS_TRUNK_FILE_INFO_LEN + 1]; + trunk_file_info_encode(&(pFileContext->extra_info.upload. \ + trunk_info.file), trunk_buff); + + sprintf(new_fname2log + FDFS_LOGIC_FILE_PATH_LEN \ + + FDFS_FILENAME_BASE64_LENGTH, "%s%s", trunk_buff, \ + new_filename + FDFS_TRUE_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH); + } + else if (rename(pFileContext->filename, new_full_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "rename %s to %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pFileContext->filename, new_full_filename, \ + result, STRERROR(result)); + + unlink(pFileContext->filename); + return result; + } + + pFileContext->timestamp2log = end_time; + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_APPENDER) + { + strcpy(pFileContext->fname2log, new_fname2log); + pFileContext->create_flag = STORAGE_CREATE_FLAG_FILE; + return 0; + } + + if ((pFileContext->extra_info.upload.file_type & _FILE_TYPE_SLAVE)) + { + char true_filename[128]; + char filename[128]; + int master_store_path_index; + int master_filename_len = strlen(pFileContext->extra_info. \ + upload.master_filename); + if ((result=storage_split_filename_ex(pFileContext->extra_info.\ + upload.master_filename, &master_filename_len, \ + true_filename, &master_store_path_index)) != 0) + { + unlink(new_full_filename); + return result; + } + if ((result=fdfs_gen_slave_filename(true_filename, \ + pFileContext->extra_info.upload.prefix_name, \ + pFileContext->extra_info.upload.file_ext_name, \ + filename, &filename_len)) != 0) + { + unlink(new_full_filename); + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[master_store_path_index], \ + filename); + sprintf(pFileContext->fname2log, \ + "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + master_store_path_index, filename); + + if (g_store_slave_file_use_link) + { + if (symlink(new_full_filename, pFileContext->filename) != 0) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "link file %s to %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, new_full_filename, \ + pFileContext->filename, \ + result, STRERROR(result)); + + unlink(new_full_filename); + return result; + } + + result = storage_binlog_write( \ + pFileContext->timestamp2log, \ + STORAGE_OP_TYPE_SOURCE_CREATE_FILE, \ + new_fname2log); + if (result == 0) + { + char binlog_buff[256]; + snprintf(binlog_buff, sizeof(binlog_buff), \ + "%s %s", pFileContext->fname2log, \ + new_fname2log); + result = storage_binlog_write( \ + pFileContext->timestamp2log, \ + STORAGE_OP_TYPE_SOURCE_CREATE_LINK, \ + binlog_buff); + } + if (result != 0) + { + unlink(new_full_filename); + unlink(pFileContext->filename); + return result; + } + + pFileContext->create_flag = STORAGE_CREATE_FLAG_LINK; + } + else + { + if (rename(new_full_filename, pFileContext->filename) != 0) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "rename file %s to %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, new_full_filename, \ + pFileContext->filename, \ + result, STRERROR(result)); + + unlink(new_full_filename); + return result; + } + + pFileContext->create_flag = STORAGE_CREATE_FLAG_FILE; + } + + return 0; + } + + strcpy(pFileContext->fname2log, new_fname2log); + if (!(pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK)) + { + strcpy(pFileContext->filename, new_full_filename); + } + + if (g_check_file_duplicate && !(pFileContext->extra_info.upload.file_type & \ + _FILE_TYPE_LINK)) + { + GroupArray *pGroupArray; + char value[128]; + FDHTKeyInfo key_info; + char *pValue; + int value_len; + int nSigLen; + char szFileSig[FILE_SIGNATURE_SIZE]; + //char buff[64]; + + memset(&key_info, 0, sizeof(key_info)); + key_info.namespace_len = g_namespace_len; + memcpy(key_info.szNameSpace, g_key_namespace, g_namespace_len); + + pGroupArray=&((g_nio_thread_data+pClientInfo->nio_thread_index)\ + ->group_array); + + STORAGE_GEN_FILE_SIGNATURE(file_size, \ + pFileContext->file_hash_codes, szFileSig) + /* + bin2hex(szFileSig, FILE_SIGNATURE_SIZE, buff); + logInfo("file: "__FILE__", line: %d, " \ + "file sig: %s", __LINE__, buff); + */ + + nSigLen = FILE_SIGNATURE_SIZE; + key_info.obj_id_len = nSigLen; + memcpy(key_info.szObjectId, szFileSig, nSigLen); + key_info.key_len = sizeof(FDHT_KEY_NAME_FILE_ID) - 1; + memcpy(key_info.szKey, FDHT_KEY_NAME_FILE_ID, \ + sizeof(FDHT_KEY_NAME_FILE_ID) - 1); + + pValue = value; + value_len = sizeof(value) - 1; + result = fdht_get_ex1(pGroupArray, g_keep_alive, \ + &key_info, FDHT_EXPIRES_NONE, \ + &pValue, &value_len, malloc); + if (result == 0) + { //exists + char *pGroupName; + char *pSrcFilename; + char *pSeperator; + + *(value + value_len) = '\0'; + pSeperator = strchr(value, '/'); + if (pSeperator == NULL) + { + logError("file: "__FILE__", line: %d, "\ + "value %s is invalid", \ + __LINE__, value); + + return EINVAL; + } + + *pSeperator = '\0'; + pGroupName = value; + pSrcFilename = pSeperator + 1; + + if ((result=storage_delete_file_auto(pFileContext)) != 0) + { + logError("file: "__FILE__", line: %d, "\ + "unlink %s fail, errno: %d, " \ + "error info: %s", __LINE__, \ + ((pFileContext->extra_info.upload. \ + file_type & _FILE_TYPE_TRUNK) ? \ + pFileContext->fname2log \ + : pFileContext->filename), \ + result, STRERROR(result)); + + return result; + } + + memset(pFileContext->extra_info.upload.group_name, \ + 0, FDFS_GROUP_NAME_MAX_LEN + 1); + snprintf(pFileContext->extra_info.upload.group_name, \ + FDFS_GROUP_NAME_MAX_LEN + 1, "%s", pGroupName); + result = storage_client_create_link_wrapper(pTask, \ + pFileContext->extra_info.upload.master_filename, \ + pSrcFilename, value_len-(pSrcFilename-value),\ + key_info.szObjectId, key_info.obj_id_len, \ + pGroupName, \ + pFileContext->extra_info.upload.prefix_name, \ + pFileContext->extra_info.upload.file_ext_name,\ + pFileContext->fname2log, &filename_len); + + pFileContext->create_flag = STORAGE_CREATE_FLAG_LINK; + return result; + } + else if (result == ENOENT) + { + char src_filename[128]; + FDHTKeyInfo ref_count_key; + + filename_len = sprintf(src_filename, "%s", new_fname2log); + value_len = sprintf(value, "%s/%s", \ + g_group_name, new_fname2log); + if ((result=fdht_set_ex(pGroupArray, g_keep_alive, \ + &key_info, FDHT_EXPIRES_NEVER, \ + value, value_len)) != 0) + { + logError("file: "__FILE__", line: %d, "\ + "client ip: %s, fdht_set fail,"\ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + + storage_delete_file_auto(pFileContext); + return result; + } + + memcpy(&ref_count_key, &key_info, sizeof(FDHTKeyInfo)); + ref_count_key.obj_id_len = value_len; + memcpy(ref_count_key.szObjectId, value, value_len); + ref_count_key.key_len = sizeof(FDHT_KEY_NAME_REF_COUNT) - 1; + memcpy(ref_count_key.szKey, FDHT_KEY_NAME_REF_COUNT, \ + ref_count_key.key_len); + if ((result=fdht_set_ex(pGroupArray, g_keep_alive, \ + &ref_count_key, FDHT_EXPIRES_NEVER, "0", 1)) != 0) + { + logError("file: "__FILE__", line: %d, "\ + "client ip: %s, fdht_set fail,"\ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + + storage_delete_file_auto(pFileContext); + return result; + } + + + result = storage_binlog_write(pFileContext->timestamp2log, \ + STORAGE_OP_TYPE_SOURCE_CREATE_FILE, \ + src_filename); + if (result != 0) + { + storage_delete_file_auto(pFileContext); + return result; + } + + result = storage_client_create_link_wrapper(pTask, \ + pFileContext->extra_info.upload.master_filename, \ + src_filename, filename_len, szFileSig, nSigLen,\ + g_group_name, pFileContext->extra_info.upload.prefix_name, \ + pFileContext->extra_info.upload.file_ext_name, \ + pFileContext->fname2log, &filename_len); + + if (result != 0) + { + fdht_delete_ex(pGroupArray, g_keep_alive, &key_info); + fdht_delete_ex(pGroupArray, g_keep_alive, &ref_count_key); + + storage_delete_file_auto(pFileContext); + } + + pFileContext->create_flag = STORAGE_CREATE_FLAG_LINK; + return result; + } + else //error + { + logError("file: "__FILE__", line: %d, " \ + "fdht_get fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(errno)); + + storage_delete_file_auto(pFileContext); + return result; + } + } + + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_LINK) + { + pFileContext->create_flag = STORAGE_CREATE_FLAG_LINK; + } + else + { + pFileContext->create_flag = STORAGE_CREATE_FLAG_FILE; + } + + return 0; +} + +static int storage_trunk_do_create_link(struct fast_task_info *pTask, \ + const int64_t file_bytes, const int buff_offset, \ + FileBeforeOpenCallback before_open_callback, + FileDealDoneCallback done_callback) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int64_t file_offset; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + file_offset = TRUNK_FILE_START_OFFSET( \ + pFileContext->extra_info.upload.trunk_info); + trunk_get_full_filename(&(pFileContext->extra_info.upload.trunk_info), + pFileContext->filename, sizeof(pFileContext->filename)); + pFileContext->extra_info.upload.before_open_callback = \ + before_open_callback; + pFileContext->extra_info.upload.before_close_callback = \ + dio_write_chunk_header; + pFileContext->open_flags = O_RDWR | g_extra_open_file_flags; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->fd = -1; + pFileContext->buff_offset = buff_offset; + pFileContext->offset = file_offset; + pFileContext->start = file_offset; + pFileContext->end = file_offset + file_bytes; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, pFileContext->extra_info.upload.trunk_info.path. \ + store_path_index, pFileContext->op); + + pFileContext->done_callback = done_callback; + pClientInfo->clean_func = dio_trunk_write_finish_clean_up; + + return dio_write_file(pTask); +} + +static int storage_trunk_create_link(struct fast_task_info *pTask, \ + const char *src_filename, const SourceFileInfo *pSourceFileInfo, \ + const bool bNeedReponse) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + FDFSTrunkFullInfo *pTrunkInfo; + TrunkCreateLinkArg *pCreateLinkArg; + char *p; + int64_t file_bytes; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + file_bytes = strlen(src_filename); + + pFileContext->extra_info.upload.if_sub_path_alloced = true; + pTrunkInfo = &(pFileContext->extra_info.upload.trunk_info); + if ((result=trunk_client_trunk_alloc_space( \ + TRUNK_CALC_SIZE(file_bytes), pTrunkInfo)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + pTask->length = pTask->size; + p = pTask->data + (pTask->length - sizeof(TrunkCreateLinkArg) \ + - file_bytes); + if (p < pTask->data + sizeof(TrackerHeader)) + { + logError("file: "__FILE__", line: %d, " \ + "task buffer size: %d is too small", \ + __LINE__, pTask->size); + pClientInfo->total_length = sizeof(TrackerHeader); + return ENOSPC; + } + + pCreateLinkArg = (TrunkCreateLinkArg *)p; + memcpy(&(pCreateLinkArg->src_file_info), pSourceFileInfo, \ + sizeof(SourceFileInfo)); + pCreateLinkArg->need_response = bNeedReponse; + pClientInfo->extra_arg = (void *)pCreateLinkArg; + p += sizeof(TrunkCreateLinkArg); + memcpy(p, src_filename, file_bytes); + + storage_trunk_do_create_link(pTask, file_bytes, p - pTask->data, \ + dio_check_trunk_file_when_upload, \ + storage_trunk_create_link_file_done_callback); + return STORAGE_STATUE_DEAL_FILE; +} + +static int storage_service_do_create_link(struct fast_task_info *pTask, \ + const SourceFileInfo *pSrcFileInfo, \ + const int64_t file_size, const char *master_filename, \ + const char *prefix_name, const char *file_ext_name, \ + char *filename, int *filename_len) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int result; + int crc32; + int store_path_index; + char src_full_filename[MAX_PATH_SIZE+64]; + char full_filename[MAX_PATH_SIZE+64]; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + store_path_index = pFileContext->extra_info. \ + upload.trunk_info.path.store_path_index; + if (*filename_len == 0) + { + char formatted_ext_name[FDFS_FILE_EXT_NAME_MAX_LEN + 2]; + + storage_format_ext_name(file_ext_name, formatted_ext_name); + crc32 = rand(); + if ((result=storage_get_filename(pClientInfo, g_current_time, \ + file_size, crc32, formatted_ext_name, filename, \ + filename_len, full_filename)) != 0) + { + return result; + } + } + else + { + sprintf(full_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + filename); + } + + sprintf(src_full_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + pSrcFileInfo->src_true_filename); + if (symlink(src_full_filename, full_filename) != 0) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "link file %s to %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + src_full_filename, full_filename, \ + result, STRERROR(result)); + *filename = '\0'; + *filename_len = 0; + return result; + } + + *filename_len=sprintf(full_filename, \ + "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + store_path_index, filename); + memcpy(filename, full_filename, (*filename_len) + 1); + + return storage_set_link_file_meta(pTask, pSrcFileInfo, filename); +} + +static int storage_set_link_file_meta(struct fast_task_info *pTask, \ + const SourceFileInfo *pSrcFileInfo, const char *link_filename) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + GroupArray *pGroupArray; + FDHTKeyInfo key_info; + char value[128]; + int value_len; + int result; + + if (!g_check_file_duplicate) + { + return 0; + } + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + memset(&key_info, 0, sizeof(key_info)); + key_info.namespace_len = g_namespace_len; + memcpy(key_info.szNameSpace, g_key_namespace, g_namespace_len); + + pGroupArray=&((g_nio_thread_data + pClientInfo->nio_thread_index) \ + ->group_array); + + key_info.obj_id_len = snprintf(key_info.szObjectId, \ + sizeof(key_info.szObjectId), \ + "%s/%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + g_group_name, FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + pFileContext->extra_info.upload.trunk_info.path. \ + store_path_index, pSrcFileInfo->src_true_filename); + + key_info.key_len = sizeof(FDHT_KEY_NAME_REF_COUNT) - 1; + memcpy(key_info.szKey, FDHT_KEY_NAME_REF_COUNT, key_info.key_len); + value_len = sizeof(value) - 1; + if ((result=fdht_inc_ex(pGroupArray, g_keep_alive, &key_info, \ + FDHT_EXPIRES_NEVER, 1, value, &value_len)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_inc fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + return 0; + } + + key_info.obj_id_len = snprintf(key_info.szObjectId, \ + sizeof(key_info.szObjectId), \ + "%s/%s", g_group_name, link_filename); + key_info.key_len = sizeof(FDHT_KEY_NAME_FILE_SIG) - 1; + memcpy(key_info.szKey, FDHT_KEY_NAME_FILE_SIG, key_info.key_len); + if ((result=fdht_set_ex(pGroupArray, g_keep_alive, \ + &key_info, FDHT_EXPIRES_NEVER, \ + pSrcFileInfo->src_file_sig, \ + pSrcFileInfo->src_file_sig_len)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, fdht_set fail," \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + result, STRERROR(result)); + } + + /* + logInfo("create link, counter=%s, object_id=%s(%d), key=%s, file_sig_len=(%d)", \ + value, key_info.szObjectId, key_info.obj_id_len, \ + FDHT_KEY_NAME_FILE_SIG, \ + pSrcFileInfo->src_file_sig_len); + */ + + return 0; +} + +static int storage_do_set_metadata(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + FDFSMetaData *old_meta_list; + FDFSMetaData *new_meta_list; + FDFSMetaData *all_meta_list; + FDFSMetaData *pOldMeta; + FDFSMetaData *pNewMeta; + FDFSMetaData *pAllMeta; + FDFSMetaData *pOldMetaEnd; + FDFSMetaData *pNewMetaEnd; + char *meta_buff; + char *file_buff; + char *all_meta_buff; + int64_t file_bytes; + int meta_bytes; + int old_meta_count; + int new_meta_count; + int all_meta_bytes; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + pFileContext->sync_flag = '\0'; + meta_buff = pFileContext->extra_info.setmeta.meta_buff; + meta_bytes = pFileContext->extra_info.setmeta.meta_bytes; + + do + { + if (pFileContext->extra_info.setmeta.op_flag == \ + STORAGE_SET_METADATA_FLAG_OVERWRITE) + { + if (meta_bytes == 0) + { + if (!fileExists(pFileContext->filename)) + { + result = 0; + break; + } + + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_DELETE_FILE; + if (unlink(pFileContext->filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, delete file %s fail," \ + "errno: %d, error info: %s", __LINE__, \ + pTask->client_ip, pFileContext->filename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EPERM; + } + else + { + result = 0; + } + + break; + } + + if ((result=storage_sort_metadata_buff(meta_buff, \ + meta_bytes)) != 0) + { + break; + } + + if (fileExists(pFileContext->filename)) + { + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_UPDATE_FILE; + } + else + { + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_CREATE_FILE; + } + + result = writeToFile(pFileContext->filename, meta_buff, meta_bytes); + break; + } + + if (meta_bytes == 0) + { + result = 0; + break; + } + + result = getFileContent(pFileContext->filename, &file_buff, &file_bytes); + if (result == ENOENT) + { + if (meta_bytes == 0) + { + result = 0; + break; + } + + if ((result=storage_sort_metadata_buff(meta_buff, \ + meta_bytes)) != 0) + { + break; + } + + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_CREATE_FILE; + result = writeToFile(pFileContext->filename, meta_buff, meta_bytes); + break; + } + else if (result != 0) + { + break; + } + + old_meta_list = fdfs_split_metadata(file_buff, &old_meta_count, &result); + if (old_meta_list == NULL) + { + free(file_buff); + break; + } + + new_meta_list = fdfs_split_metadata(meta_buff, &new_meta_count, &result); + if (new_meta_list == NULL) + { + free(file_buff); + free(old_meta_list); + break; + } + + all_meta_list = (FDFSMetaData *)malloc(sizeof(FDFSMetaData) * \ + (old_meta_count + new_meta_count)); + if (all_meta_list == NULL) + { + free(file_buff); + free(old_meta_list); + free(new_meta_list); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSMetaData) \ + * (old_meta_count + new_meta_count)); + result = errno != 0 ? errno : ENOMEM; + break; + } + + qsort((void *)new_meta_list, new_meta_count, sizeof(FDFSMetaData), \ + metadata_cmp_by_name); + + pOldMetaEnd = old_meta_list + old_meta_count; + pNewMetaEnd = new_meta_list + new_meta_count; + pOldMeta = old_meta_list; + pNewMeta = new_meta_list; + pAllMeta = all_meta_list; + while (pOldMeta < pOldMetaEnd && pNewMeta < pNewMetaEnd) + { + result = strcmp(pOldMeta->name, pNewMeta->name); + if (result < 0) + { + memcpy(pAllMeta, pOldMeta, sizeof(FDFSMetaData)); + pOldMeta++; + } + else if (result == 0) + { + memcpy(pAllMeta, pNewMeta, sizeof(FDFSMetaData)); + pOldMeta++; + pNewMeta++; + } + else //result > 0 + { + memcpy(pAllMeta, pNewMeta, sizeof(FDFSMetaData)); + pNewMeta++; + } + + pAllMeta++; + } + + while (pOldMeta < pOldMetaEnd) + { + memcpy(pAllMeta, pOldMeta, sizeof(FDFSMetaData)); + pOldMeta++; + pAllMeta++; + } + + while (pNewMeta < pNewMetaEnd) + { + memcpy(pAllMeta, pNewMeta, sizeof(FDFSMetaData)); + pNewMeta++; + pAllMeta++; + } + + free(file_buff); + free(old_meta_list); + free(new_meta_list); + + all_meta_buff = fdfs_pack_metadata(all_meta_list, \ + pAllMeta - all_meta_list, NULL, &all_meta_bytes); + free(all_meta_list); + if (all_meta_buff == NULL) + { + result = errno != 0 ? errno : ENOMEM; + break; + } + + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_UPDATE_FILE; + result = writeToFile(pFileContext->filename, all_meta_buff, all_meta_bytes); + + free(all_meta_buff); + } while (0); + + storage_set_metadata_done_callback(pTask, result); + return result; +} + +/** +8 bytes: filename length +8 bytes: meta data size +1 bytes: operation flag, + 'O' for overwrite all old metadata + 'M' for merge, insert when the meta item not exist, otherwise update it +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +meta data bytes: each meta data seperated by \x01, + name and value seperated by \x02 +**/ +static int storage_server_set_metadata(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int64_t nInPackLen; + FDFSTrunkHeader trunkHeader; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char filename[128]; + char true_filename[128]; + char *p; + char *meta_buff; + int meta_bytes; + int filename_len; + int true_filename_len; + int result; + int store_path_index; + struct stat stat_buf; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 2 * FDFS_PROTO_PKG_LEN_SIZE + 1 + \ + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", \ + __LINE__, STORAGE_PROTO_CMD_SET_METADATA, \ + pTask->client_ip, nInPackLen, \ + 2 * FDFS_PROTO_PKG_LEN_SIZE + 1 \ + + FDFS_GROUP_NAME_MAX_LEN); + + return EINVAL; + } + + if (nInPackLen + sizeof(TrackerHeader) >= pTask->size) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length < %d", \ + __LINE__, STORAGE_PROTO_CMD_SET_METADATA, \ + pTask->client_ip, nInPackLen, \ + pTask->size - (int)sizeof(TrackerHeader)); + + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + meta_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (filename_len <= 0 || filename_len >= sizeof(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid filename length: %d", \ + __LINE__, pTask->client_ip, filename_len); + + return EINVAL; + } + + pFileContext->extra_info.setmeta.op_flag = *p++; + if (pFileContext->extra_info.setmeta.op_flag != \ + STORAGE_SET_METADATA_FLAG_OVERWRITE && \ + pFileContext->extra_info.setmeta.op_flag != \ + STORAGE_SET_METADATA_FLAG_MERGE) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, " \ + "invalid operation flag: 0x%02X", \ + __LINE__, pTask->client_ip, \ + pFileContext->extra_info.setmeta.op_flag); + + return EINVAL; + } + + if (meta_bytes < 0 || meta_bytes != nInPackLen - \ + (2 * FDFS_PROTO_PKG_LEN_SIZE + 1 + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid meta bytes: %d", \ + __LINE__, pTask->client_ip, meta_bytes); + + return EINVAL; + } + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + memcpy(filename, p, filename_len); + *(filename + filename_len) = '\0'; + p += filename_len; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(filename, filename_len, \ + pClientInfo); + + true_filename_len = filename_len; + if ((result=storage_split_filename_ex(filename, \ + &true_filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, \ + true_filename_len)) != 0) + { + return result; + } + + meta_buff = p; + *(meta_buff + meta_bytes) = '\0'; + + if ((result=trunk_file_lstat(store_path_index, true_filename, \ + true_filename_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &trunkHeader)) != 0) + { + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "logic", filename) + return result; + } + + pFileContext->timestamp2log = g_current_time; + sprintf(pFileContext->filename, "%s/data/%s%s", \ + g_fdfs_store_paths.paths[store_path_index], true_filename, \ + FDFS_STORAGE_META_FILE_EXT); + sprintf(pFileContext->fname2log,"%s%s", \ + filename, FDFS_STORAGE_META_FILE_EXT); + + pClientInfo->deal_func = storage_do_set_metadata; + pFileContext->extra_info.setmeta.meta_buff = meta_buff; + pFileContext->extra_info.setmeta.meta_bytes = meta_bytes; + + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, store_path_index, FDFS_STORAGE_FILE_OP_WRITE); + + if ((result=storage_dio_queue_push(pTask)) != 0) + { + return result; + } + + return STORAGE_STATUE_DEAL_FILE; +} + +/** +IP_ADDRESS_SIZE bytes: tracker client ip address +**/ +static int storage_server_report_server_id(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + char *storage_server_id; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + if (nInPackLen != FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + STORAGE_PROTO_CMD_REPORT_SERVER_ID, \ + pTask->client_ip, nInPackLen, \ + FDFS_STORAGE_ID_MAX_SIZE); + return EINVAL; + } + + storage_server_id = pTask->data + sizeof(TrackerHeader); + *(storage_server_id + (FDFS_STORAGE_ID_MAX_SIZE - 1)) = '\0'; + if (*storage_server_id == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, storage server id is empty!", \ + __LINE__, pTask->client_ip); + return EINVAL; + } + + strcpy(pClientInfo->storage_server_id, storage_server_id); + + logDebug("file: "__FILE__", line: %d, " \ + "client ip: %s, storage server id: %s", \ + __LINE__, pTask->client_ip, storage_server_id); + + return 0; +} + +/** +N bytes: binlog +**/ +static int storage_server_trunk_sync_binlog(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + char *binlog_buff; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + if (nInPackLen == 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct", __LINE__, \ + STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG, \ + pTask->client_ip, nInPackLen); + return EINVAL; + } + + if (!g_if_use_trunk_file) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid command: %d, " \ + "because i don't use trunk file!", \ + __LINE__, pTask->client_ip, \ + STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG); + return EINVAL; + } + + if (g_if_trunker_self) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid command: %d, " \ + "because i am the TRUNK server!", \ + __LINE__, pTask->client_ip, \ + STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG); + return EINVAL; + } + + binlog_buff = pTask->data + sizeof(TrackerHeader); + return trunk_binlog_write_buffer(binlog_buff, nInPackLen); +} + +/** +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +**/ +static int storage_server_query_file_info(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + char *in_buff; + char *filename; + char *p; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char src_filename[MAX_PATH_SIZE + 128]; + char decode_buff[64]; + struct stat file_lstat; + struct stat file_stat; + FDFSTrunkFullInfo trunkInfo; + FDFSTrunkHeader trunkHeader; + int64_t nInPackLen; + int store_path_index; + int filename_len; + int true_filename_len; + int crc32; + int storage_id; + int result; + int len; + int buff_len; + bool bSilence; + + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + if (nInPackLen <= FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_QUERY_FILE_INFO, \ + pTask->client_ip, nInPackLen, \ + FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + filename_len = nInPackLen - FDFS_GROUP_NAME_MAX_LEN; + if (filename_len >= sizeof(pClientInfo->file_context.fname2log)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, filename length: %d" \ + " is not correct, expect length < %d", __LINE__, \ + STORAGE_PROTO_CMD_QUERY_FILE_INFO, \ + pTask->client_ip, filename_len, \ + (int)sizeof(pClientInfo->file_context.fname2log)); + return EINVAL; + } + + in_buff = pTask->data + sizeof(TrackerHeader); + filename = in_buff + FDFS_GROUP_NAME_MAX_LEN; + *(filename + filename_len) = '\0'; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(filename, filename_len, \ + pClientInfo); + + bSilence = ((TrackerHeader *)pTask->data)->status != 0; + memcpy(group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + true_filename_len = filename_len; + if ((result=storage_split_filename_ex(filename, &true_filename_len, \ + true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, \ + true_filename_len)) != 0) + { + return result; + } + + if ((result=trunk_file_lstat(store_path_index, true_filename, \ + true_filename_len, &file_lstat, \ + &trunkInfo, &trunkHeader)) != 0) + { + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, lstat logic file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, filename, \ + result, STRERROR(result)); + } + else if (!bSilence) + { + logDebug("file: "__FILE__", line: %d, " \ + "client ip:%s, logic file: %s not exist", \ + __LINE__, pTask->client_ip, filename); + } + + return result; + } + + if (S_ISLNK(file_lstat.st_mode)) + { + if (IS_TRUNK_FILE_BY_ID(trunkInfo)) + { + char src_true_filename[128]; + int src_filename_len; + int src_store_path_index; + + result = trunk_file_get_content(&trunkInfo, file_lstat.st_size, \ + NULL, src_filename, sizeof(src_filename) - 1); + if (result != 0) + { + if (!bSilence) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, call readlink file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, true_filename, + result, STRERROR(result)); + } + return result; + } + + src_filename_len = file_lstat.st_size; + *(src_filename + src_filename_len) = '\0'; + if ((result=storage_split_filename_ex(src_filename, \ + &src_filename_len, src_true_filename, \ + &src_store_path_index)) != 0) + { + return result; + } + + result = trunk_file_lstat(src_store_path_index, \ + src_true_filename, src_filename_len, \ + &file_stat, &trunkInfo, &trunkHeader); + if (result != 0) + { + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, call lstat logic file: %s " \ + "fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, src_filename, \ + result, STRERROR(result)); + } + else if (!bSilence) + { + logDebug("file: "__FILE__", line: %d, " \ + "client ip:%s, logic file: %s not exist", \ + __LINE__, pTask->client_ip, src_filename); + } + return result; + } + } + else + { + char full_filename[MAX_PATH_SIZE + 128]; + + sprintf(full_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + if ((len=readlink(full_filename, src_filename, \ + sizeof(src_filename))) < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, call readlink file %s " \ + "fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + true_filename, result, STRERROR(result)); + return result; + } + + *(src_filename + len) = '\0'; + strcpy(full_filename, src_filename); + if (stat(full_filename, &file_stat) != 0) + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, + pTask->client_ip, "regular", full_filename) + return result; + } + } + } + else + { + memcpy(&file_stat, &file_lstat, sizeof(struct stat)); + } + + if (filename_len < FDFS_LOGIC_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, length of filename: %s " \ + "is too small, should >= %d", \ + __LINE__, pTask->client_ip, filename, \ + FDFS_LOGIC_FILE_PATH_LEN + FDFS_FILENAME_BASE64_LENGTH); + return EINVAL; + } + + memset(decode_buff, 0, sizeof(decode_buff)); + base64_decode_auto(&g_fdfs_base64_context, filename + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + decode_buff, &buff_len); + storage_id = ntohl(buff2int(decode_buff)); + crc32 = buff2int(decode_buff + sizeof(int) * 4); + + p = pTask->data + sizeof(TrackerHeader); + long2buff(file_stat.st_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + long2buff(file_lstat.st_mtime, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + long2buff(crc32, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + memset(p, 0, IP_ADDRESS_SIZE); + if (fdfs_get_server_id_type(storage_id) == FDFS_ID_TYPE_SERVER_ID) + { + if (g_use_storage_id) + { + FDFSStorageIdInfo *pStorageIdInfo; + char id[16]; + + sprintf(id, "%d", storage_id); + pStorageIdInfo = fdfs_get_storage_by_id(id); + if (pStorageIdInfo != NULL) + { + strcpy(p, pStorageIdInfo->ip_addr); + } + } + } + else + { + struct in_addr ip_addr; + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = storage_id; + inet_ntop(AF_INET, &ip_addr, p, IP_ADDRESS_SIZE); + } + p += IP_ADDRESS_SIZE; + + pClientInfo->total_length = p - pTask->data; + return 0; +} + +#define CHECK_TRUNK_SERVER(pTask) \ + if (!g_if_trunker_self) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, i am not trunk server!", \ + __LINE__, pTask->client_ip); \ + return EINVAL; \ + } + +/** +request package format: +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +4 bytes: file size +1 bytes: store_path_index + +response package format: +1 byte: store_path_index +1 byte: sub_path_high +1 byte: sub_path_low +4 bytes: trunk file id +4 bytes: trunk offset +4 bytes: trunk size +**/ +static int storage_server_trunk_alloc_space(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + FDFSTrunkInfoBuff *pApplyBody; + char *in_buff; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSTrunkFullInfo trunkInfo; + int64_t nInPackLen; + int file_size; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + CHECK_TRUNK_SERVER(pTask) + + if (nInPackLen != FDFS_GROUP_NAME_MAX_LEN + 5) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + STORAGE_PROTO_CMD_TRUNK_ALLOC_SPACE, \ + pTask->client_ip, nInPackLen, \ + FDFS_GROUP_NAME_MAX_LEN + 5); + return EINVAL; + } + + in_buff = pTask->data + sizeof(TrackerHeader); + memcpy(group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + file_size = buff2int(in_buff + FDFS_GROUP_NAME_MAX_LEN); + if (file_size < 0 || !trunk_check_size(file_size)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid file size: %d", \ + __LINE__, pTask->client_ip, file_size); + return EINVAL; + } + + trunkInfo.path.store_path_index = *(in_buff+FDFS_GROUP_NAME_MAX_LEN+4); + if ((result=trunk_alloc_space(file_size, &trunkInfo)) != 0) + { + return result; + } + + pApplyBody = (FDFSTrunkInfoBuff *)(pTask->data+sizeof(TrackerHeader)); + pApplyBody->store_path_index = trunkInfo.path.store_path_index; + pApplyBody->sub_path_high = trunkInfo.path.sub_path_high; + pApplyBody->sub_path_low = trunkInfo.path.sub_path_low; + int2buff(trunkInfo.file.id, pApplyBody->id); + int2buff(trunkInfo.file.offset, pApplyBody->offset); + int2buff(trunkInfo.file.size, pApplyBody->size); + + pClientInfo->total_length = sizeof(TrackerHeader) + \ + sizeof(FDFSTrunkInfoBuff); + return 0; +} + +#define storage_server_trunk_alloc_confirm(pTask) \ + storage_server_trunk_confirm_or_free(pTask) + +#define storage_server_trunk_free_space(pTask) \ + storage_server_trunk_confirm_or_free(pTask) + +static int storage_server_trunk_get_binlog_size(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + TrackerHeader *pHeader; + char *p; + char binlog_filename[MAX_PATH_SIZE]; + struct stat file_stat; + int64_t nInPackLen; + + pHeader = (TrackerHeader *)pTask->data; + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + + if (nInPackLen != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length: 0", __LINE__, \ + pHeader->cmd, pTask->client_ip, nInPackLen); + pClientInfo->total_length = sizeof (TrackerHeader); + return EINVAL; + } + + if (!g_if_use_trunk_file) + { + logError ("file: " __FILE__ ", line: %d, " + "client ip: %s, i don't support trunked file!", \ + __LINE__, pTask->client_ip); + pClientInfo->total_length = sizeof (TrackerHeader); + return EINVAL; + } + + get_trunk_binlog_filename(binlog_filename); + if (stat(binlog_filename, &file_stat) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, " \ + "stat trunk binlog file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pHeader->cmd, pTask->client_ip, + binlog_filename, errno, STRERROR(errno)); + pClientInfo->total_length = sizeof (TrackerHeader); + return errno != 0 ? errno : ENOENT; + } + + p = pTask->data + sizeof(TrackerHeader); + long2buff(file_stat.st_size, p); + + pClientInfo->total_length = sizeof(TrackerHeader) + + FDFS_PROTO_PKG_LEN_SIZE; + return 0; +} + +static int storage_server_trunk_truncate_binlog_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + TrackerHeader *pHeader; + int64_t nInPackLen; + + pHeader = (TrackerHeader *)pTask->data; + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof (TrackerHeader); + + if (nInPackLen != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length: 0", __LINE__, \ + pHeader->cmd, pTask->client_ip, nInPackLen); + return EINVAL; + } + + if (!g_if_use_trunk_file) + { + logError ("file: " __FILE__ ", line: %d, " + "client ip: %s, i don't support trunked file!", \ + __LINE__, pTask->client_ip); + return EINVAL; + } + + if (g_if_trunker_self) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid command: %d, " \ + "because i am the TRUNK server!", \ + __LINE__, pTask->client_ip, pHeader->cmd); + return EINVAL; + } + + return trunk_binlog_truncate(); +} + +static int storage_server_trunk_delete_binlog_marks(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + TrackerHeader *pHeader; + int64_t nInPackLen; + int result; + + pHeader = (TrackerHeader *)pTask->data; + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof (TrackerHeader); + + if (nInPackLen != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length: 0", __LINE__, \ + pHeader->cmd, pTask->client_ip, nInPackLen); + return EINVAL; + } + + if (!g_if_use_trunk_file) + { + logError ("file: " __FILE__ ", line: %d, " + "client ip: %s, i don't support trunked file!", \ + __LINE__, pTask->client_ip); + return EINVAL; + } + + if (g_if_trunker_self) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid command: %d, " \ + "because i am the TRUNK server!", \ + __LINE__, pTask->client_ip, pHeader->cmd); + return EINVAL; + } + + result = storage_delete_trunk_data_file(); + if (!(result == 0 || result == ENOENT)) + { + return result; + } + + return trunk_unlink_all_mark_files(); +} + +/** +request package format: + FDFS_GROUP_NAME_MAX_LEN bytes: group_name + 1 byte: store_path_index + 1 byte: sub_path_high + 1 byte: sub_path_low + 4 bytes: trunk file id + 4 bytes: trunk offset + 4 bytes: trunk size +**/ +static int storage_server_trunk_confirm_or_free(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + TrackerHeader *pHeader; + FDFSTrunkInfoBuff *pTrunkBuff; + char *in_buff; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSTrunkFullInfo trunkInfo; + int64_t nInPackLen; + + pHeader = (TrackerHeader *)pTask->data; + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + CHECK_TRUNK_SERVER(pTask) + + if (nInPackLen != STORAGE_TRUNK_ALLOC_CONFIRM_REQ_BODY_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + pHeader->cmd, pTask->client_ip, nInPackLen, \ + (int)STORAGE_TRUNK_ALLOC_CONFIRM_REQ_BODY_LEN); + return EINVAL; + } + + in_buff = pTask->data + sizeof(TrackerHeader); + memcpy(group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + pTrunkBuff = (FDFSTrunkInfoBuff *)(in_buff + FDFS_GROUP_NAME_MAX_LEN); + trunkInfo.path.store_path_index = pTrunkBuff->store_path_index; + trunkInfo.path.sub_path_high = pTrunkBuff->sub_path_high; + trunkInfo.path.sub_path_low = pTrunkBuff->sub_path_low; + trunkInfo.file.id = buff2int(pTrunkBuff->id); + trunkInfo.file.offset = buff2int(pTrunkBuff->offset); + trunkInfo.file.size = buff2int(pTrunkBuff->size); + + if (pHeader->cmd == STORAGE_PROTO_CMD_TRUNK_ALLOC_CONFIRM) + { + return trunk_alloc_confirm(&trunkInfo, pHeader->status); + } + else + { + return trunk_free_space(&trunkInfo, true); + } +} + +static int storage_server_fetch_one_path_binlog_dealer( \ + struct fast_task_info *pTask) +{ +#define STORAGE_LAST_AHEAD_BYTES (2 * FDFS_PROTO_PKG_LEN_SIZE) + + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + StorageBinLogReader *pReader; + char *pOutBuff; + char *pBasePath; + int result; + int record_len; + int base_path_len; + int len; + int store_path_index; + struct stat stat_buf; + char diskLogicPath[16]; + char full_filename[MAX_PATH_SIZE]; + char src_filename[MAX_PATH_SIZE]; + bool bLast; + StorageBinLogRecord record; + int64_t pkg_len; + + pClientInfo = (StorageClientInfo *)pTask->arg; + if (pClientInfo->total_length - pClientInfo->total_offset <= \ + STORAGE_LAST_AHEAD_BYTES) //finished, close the connection + { + STORAGE_NIO_NOTIFY_CLOSE(pTask); + return 0; + } + + pFileContext = &(pClientInfo->file_context); + pReader = (StorageBinLogReader *)pClientInfo->extra_arg; + + store_path_index = pFileContext->extra_info.upload.trunk_info. \ + path.store_path_index; + pBasePath = g_fdfs_store_paths.paths[store_path_index]; + base_path_len = strlen(pBasePath); + pOutBuff = pTask->data; + + bLast = false; + sprintf(diskLogicPath, "%c"FDFS_STORAGE_DATA_DIR_FORMAT, \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, store_path_index); + + do + { + result = storage_binlog_read(pReader, &record, &record_len); + if (result == ENOENT) //no binlog record + { + bLast = true; + result = 0; + break; + } + else if (result != 0) + { + break; + } + + if (g_fdfs_store_paths.paths[record.store_path_index] != pBasePath) + { + continue; + } + + if (!(record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_FILE + || record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_LINK)) + { + continue; + } + + if (fdfs_is_trunk_file(record.filename, record.filename_len)) + { + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK) + { + record.op_type = STORAGE_OP_TYPE_SOURCE_CREATE_FILE; + } + else if (record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_LINK) + { + record.op_type = STORAGE_OP_TYPE_REPLICA_CREATE_FILE; + } + } + else + { + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + g_fdfs_store_paths.paths[record.store_path_index], \ + record.true_filename); + if (lstat(full_filename, &stat_buf) != 0) + { + if (errno == ENOENT) + { + continue; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "call stat fail, file: %s, "\ + "error no: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EPERM; + break; + } + } + + if (S_ISLNK(stat_buf.st_mode)) + { + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_FILE) + { + logWarning("file: "__FILE__", line: %d, " \ + "regular file %s change to symbol " \ + "link file, some mistake happen?", \ + __LINE__, full_filename); + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE) + { + record.op_type = STORAGE_OP_TYPE_SOURCE_CREATE_LINK; + } + else + { + record.op_type = STORAGE_OP_TYPE_REPLICA_CREATE_LINK; + } + } + } + else + { + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_LINK) + { + logWarning("file: "__FILE__", line: %d, " \ + "symbol link file %s change to " \ + "regular file, some mistake happen?", \ + __LINE__, full_filename); + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK) + { + record.op_type = STORAGE_OP_TYPE_SOURCE_CREATE_FILE; + } + else + { + record.op_type = STORAGE_OP_TYPE_REPLICA_CREATE_FILE; + } + } + } + } + + if (record.op_type == STORAGE_OP_TYPE_SOURCE_CREATE_FILE + || record.op_type == STORAGE_OP_TYPE_REPLICA_CREATE_FILE) + { + pOutBuff += sprintf(pOutBuff, "%d %c %s\n", \ + (int)record.timestamp, \ + record.op_type, record.filename); + } + else + { + if ((len=readlink(full_filename, src_filename, \ + sizeof(src_filename))) < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, call readlink file " \ + "%s fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + full_filename, result, STRERROR(result)); + + if (result == ENOENT) + { + continue; + } + break; + } + + if (len <= base_path_len) + { + logWarning("file: "__FILE__", line: %d, " \ + "invalid symbol link file: %s, " \ + "maybe not create by FastDFS?", \ + __LINE__, full_filename); + continue; + } + *(src_filename + len) = '\0'; + if (!fileExists(src_filename)) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, symbol link file: %s, "\ + "it's source file: %s not exist", \ + __LINE__, pTask->client_ip, \ + full_filename, src_filename); + continue; + } + + //full filename format: ${base_path}/data/filename + pOutBuff += sprintf(pOutBuff, "%d %c %s %s/%s\n", \ + (int)record.timestamp, \ + record.op_type, record.filename, \ + diskLogicPath, \ + src_filename + base_path_len + 6); + } + + if (pTask->size - (pOutBuff - pTask->data) < \ + STORAGE_BINLOG_LINE_SIZE + FDFS_PROTO_PKG_LEN_SIZE) + { + break; + } + } while(1); + + if (result != 0) //error occurs + { + STORAGE_NIO_NOTIFY_CLOSE(pTask); + return result; + } + + pTask->length = pOutBuff - pTask->data; + if (bLast) + { + pkg_len = pClientInfo->total_offset + pTask->length - \ + sizeof(TrackerHeader); + long2buff(pkg_len, pOutBuff); + + pTask->length += FDFS_PROTO_PKG_LEN_SIZE; + pClientInfo->total_length = pkg_len + FDFS_PROTO_PKG_LEN_SIZE \ + + STORAGE_LAST_AHEAD_BYTES; + } + + storage_nio_notify(pTask); + return 0; +} + +static void fetch_one_path_binlog_finish_clean_up(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageBinLogReader *pReader; + char full_filename[MAX_PATH_SIZE]; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pReader = (StorageBinLogReader *)pClientInfo->extra_arg; + if (pReader == NULL) + { + return; + } + + pClientInfo->extra_arg = NULL; + + storage_reader_destroy(pReader); + get_mark_filename_by_reader(pReader, full_filename); + if (fileExists(full_filename)) + { + unlink(full_filename); + } + + free(pReader); +} + +static int storage_server_do_fetch_one_path_binlog( \ + struct fast_task_info *pTask, const int store_path_index) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + StorageBinLogReader *pReader; + TrackerHeader *pHeader; + int result; + + pReader = (StorageBinLogReader *)malloc(sizeof(StorageBinLogReader)); + if (pReader == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)sizeof(StorageBinLogReader), + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + if ((result=storage_reader_init(NULL, pReader)) != 0) + { + storage_reader_destroy(pReader); + return result; + } + + pClientInfo->deal_func = storage_server_fetch_one_path_binlog_dealer; + pClientInfo->clean_func = fetch_one_path_binlog_finish_clean_up; + + pFileContext->fd = -1; + pFileContext->op = FDFS_STORAGE_FILE_OP_READ; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, store_path_index, pFileContext->op); + pFileContext->extra_info.upload.trunk_info.path.store_path_index = + store_path_index; + pClientInfo->extra_arg = pReader; + + pClientInfo->total_length = INFINITE_FILE_SIZE + \ + sizeof(TrackerHeader); + pClientInfo->total_offset = 0; + pTask->length = sizeof(TrackerHeader); + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = 0; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); + + return STORAGE_STATUE_DEAL_FILE; +} + +/** +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +1 byte: store path index +**/ +static int storage_server_fetch_one_path_binlog(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + char *in_buff; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + int store_path_index; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + if (nInPackLen != FDFS_GROUP_NAME_MAX_LEN + 1) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length = %d", __LINE__, \ + STORAGE_PROTO_CMD_FETCH_ONE_PATH_BINLOG, \ + pTask->client_ip, \ + nInPackLen, FDFS_GROUP_NAME_MAX_LEN + 1); + return EINVAL; + } + + in_buff = pTask->data + sizeof(TrackerHeader); + memcpy(group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + store_path_index = *(in_buff + FDFS_GROUP_NAME_MAX_LEN); + if (store_path_index < 0 || store_path_index >= g_fdfs_store_paths.count) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, store_path_index: %d " \ + "is invalid", __LINE__, \ + pTask->client_ip, store_path_index); + return EINVAL; + } + + return storage_server_do_fetch_one_path_binlog( \ + pTask, store_path_index); +} + +/** +1 byte: store path index +8 bytes: file size +FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.) +file size bytes: file content +**/ +static int storage_upload_file(struct fast_task_info *pTask, bool bAppenderFile) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + DisconnectCleanFunc clean_func; + char *p; + char filename[128]; + char file_ext_name[FDFS_FILE_PREFIX_MAX_LEN + 1]; + int64_t nInPackLen; + int64_t file_offset; + int64_t file_bytes; + int crc32; + int store_path_index; + int result; + int filename_len; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + + if (nInPackLen < 1 + FDFS_PROTO_PKG_LEN_SIZE + + FDFS_FILE_EXT_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length >= %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, nInPackLen, \ + 1 + FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_FILE_EXT_NAME_MAX_LEN); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + store_path_index = *p++; + + if (store_path_index == -1) + { + if ((result=storage_get_storage_path_index( \ + &store_path_index)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "get_storage_path_index fail, " \ + "errno: %d, error info: %s", __LINE__, \ + result, STRERROR(result)); + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + } + else if (store_path_index < 0 || store_path_index >= \ + g_fdfs_store_paths.count) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, store_path_index: %d " \ + "is invalid", __LINE__, \ + pTask->client_ip, store_path_index); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + file_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (file_bytes < 0 || file_bytes != nInPackLen - \ + (1 + FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_FILE_EXT_NAME_MAX_LEN)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid file bytes: "INT64_PRINTF_FORMAT \ + ", total body length: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, file_bytes, nInPackLen); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(file_ext_name, p, FDFS_FILE_EXT_NAME_MAX_LEN); + *(file_ext_name + FDFS_FILE_EXT_NAME_MAX_LEN) = '\0'; + p += FDFS_FILE_EXT_NAME_MAX_LEN; + if ((result=fdfs_validate_filename(file_ext_name)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file_ext_name: %s " \ + "is invalid!", __LINE__, \ + pTask->client_ip, file_ext_name); + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + pFileContext->calc_crc32 = true; + pFileContext->calc_file_hash = g_check_file_duplicate; + pFileContext->extra_info.upload.start_time = g_current_time; + + strcpy(pFileContext->extra_info.upload.file_ext_name, file_ext_name); + storage_format_ext_name(file_ext_name, \ + pFileContext->extra_info.upload.formatted_ext_name); + pFileContext->extra_info.upload.trunk_info.path. \ + store_path_index = store_path_index; + pFileContext->extra_info.upload.file_type = _FILE_TYPE_REGULAR; + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_CREATE_FILE; + pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + if (bAppenderFile) + { + pFileContext->extra_info.upload.file_type |= \ + _FILE_TYPE_APPENDER; + } + else + { + if (g_if_use_trunk_file && trunk_check_size( \ + TRUNK_CALC_SIZE(file_bytes))) + { + pFileContext->extra_info.upload.file_type |= \ + _FILE_TYPE_TRUNK; + } + } + + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + FDFSTrunkFullInfo *pTrunkInfo; + + pFileContext->extra_info.upload.if_sub_path_alloced = true; + pTrunkInfo = &(pFileContext->extra_info.upload.trunk_info); + if ((result=trunk_client_trunk_alloc_space( \ + TRUNK_CALC_SIZE(file_bytes), pTrunkInfo)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + clean_func = dio_trunk_write_finish_clean_up; + file_offset = TRUNK_FILE_START_OFFSET((*pTrunkInfo)); + pFileContext->extra_info.upload.if_gen_filename = true; + trunk_get_full_filename(pTrunkInfo, pFileContext->filename, \ + sizeof(pFileContext->filename)); + pFileContext->extra_info.upload.before_open_callback = \ + dio_check_trunk_file_when_upload; + pFileContext->extra_info.upload.before_close_callback = \ + dio_write_chunk_header; + pFileContext->open_flags = O_RDWR | g_extra_open_file_flags; + } + else + { + char reserved_space_str[32]; + + if (!storage_check_reserved_space_path(g_path_space_list \ + [store_path_index].total_mb, g_path_space_list \ + [store_path_index].free_mb - (file_bytes/FDFS_ONE_MB), \ + g_avg_storage_reserved_mb)) + { + logError("file: "__FILE__", line: %d, " \ + "no space to upload file, " + "free space: %d MB is too small, file bytes: " \ + INT64_PRINTF_FORMAT", reserved space: %s", \ + __LINE__, g_path_space_list[store_path_index].\ + free_mb, file_bytes, \ + fdfs_storage_reserved_space_to_string_ex( \ + g_storage_reserved_space.flag, \ + g_avg_storage_reserved_mb, \ + g_path_space_list[store_path_index]. \ + total_mb, g_storage_reserved_space.rs.ratio,\ + reserved_space_str)); + pClientInfo->total_length = sizeof(TrackerHeader); + return ENOSPC; + } + + crc32 = rand(); + *filename = '\0'; + filename_len = 0; + pFileContext->extra_info.upload.if_sub_path_alloced = false; + if ((result=storage_get_filename(pClientInfo, \ + pFileContext->extra_info.upload.start_time, \ + file_bytes, crc32, pFileContext->extra_info.upload.\ + formatted_ext_name, filename, &filename_len, \ + pFileContext->filename)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + clean_func = dio_write_finish_clean_up; + file_offset = 0; + pFileContext->extra_info.upload.if_gen_filename = true; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + pFileContext->open_flags = O_WRONLY | O_CREAT | O_TRUNC \ + | g_extra_open_file_flags; + } + + return storage_write_to_file(pTask, file_offset, file_bytes, \ + p - pTask->data, dio_write_file, \ + storage_upload_file_done_callback, \ + clean_func, store_path_index); +} + +static int storage_deal_active_test(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + if (nInPackLen != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length 0", __LINE__, \ + FDFS_PROTO_CMD_ACTIVE_TEST, pTask->client_ip, \ + nInPackLen); + return EINVAL; + } + + return 0; +} + +/** +8 bytes: appender filename length +8 bytes: file size +appender filename bytes: appender filename +file size bytes: file content +**/ +static int storage_append_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + char appender_filename[128]; + char true_filename[128]; + char decode_buff[64]; + struct stat stat_buf; + int appender_filename_len; + int64_t nInPackLen; + int64_t file_bytes; + int64_t appender_file_size; + int result; + int store_path_index; + int filename_len; + int buff_len; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 2 * FDFS_PROTO_PKG_LEN_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_APPEND_FILE, \ + pTask->client_ip, \ + nInPackLen, 2 * FDFS_PROTO_PKG_LEN_SIZE); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + appender_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + file_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (appender_filename_len < FDFS_LOGIC_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH + FDFS_FILE_EXT_NAME_MAX_LEN + 1 \ + || appender_filename_len >= sizeof(appender_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid appender_filename " \ + "bytes: %d", __LINE__, \ + pTask->client_ip, appender_filename_len); + return EINVAL; + } + + if (file_bytes < 0 || file_bytes != nInPackLen - \ + (2 * FDFS_PROTO_PKG_LEN_SIZE + appender_filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid file bytes: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, file_bytes); + return EINVAL; + } + + memcpy(appender_filename, p, appender_filename_len); + *(appender_filename + appender_filename_len) = '\0'; + p += appender_filename_len; + filename_len = appender_filename_len; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(appender_filename, \ + appender_filename_len, pClientInfo); + + if ((result=storage_split_filename_ex(appender_filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], true_filename); + if (lstat(pFileContext->filename, &stat_buf) == 0) + { + if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file: %s " \ + "is not a regular file", __LINE__, \ + pTask->client_ip, pFileContext->filename); + + return EINVAL; + } + } + else + { + result = errno != 0 ? errno : ENOENT; + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file: %s " \ + "not exist", __LINE__, \ + pTask->client_ip, pFileContext->filename); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat appednder file %s fail" \ + ", errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + result, STRERROR(result)); + } + + return result; + } + + strcpy(pFileContext->fname2log, appender_filename); + + memset(decode_buff, 0, sizeof(decode_buff)); + base64_decode_auto(&g_fdfs_base64_context, pFileContext->fname2log + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + decode_buff, &buff_len); + + appender_file_size = buff2long(decode_buff + sizeof(int) * 2); + if (!IS_APPENDER_FILE(appender_file_size)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file: %s is not a valid " \ + "appender file, file size: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, appender_filename, \ + appender_file_size); + + return EINVAL; + } + + if (file_bytes == 0) + { + return 0; + } + + pFileContext->extra_info.upload.start_time = g_current_time; + pFileContext->extra_info.upload.if_gen_filename = false; + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_APPEND_FILE; + pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time; + pFileContext->extra_info.upload.file_type = _FILE_TYPE_APPENDER; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + pFileContext->extra_info.upload.trunk_info.path.store_path_index = \ + store_path_index; + pFileContext->op = FDFS_STORAGE_FILE_OP_APPEND; + pFileContext->open_flags = O_WRONLY | O_APPEND | g_extra_open_file_flags; + + return storage_write_to_file(pTask, stat_buf.st_size, file_bytes, \ + p - pTask->data, dio_write_file, \ + storage_append_file_done_callback, \ + dio_append_finish_clean_up, store_path_index); +} + +/** +8 bytes: appender filename length +8 bytes: file offset +8 bytes: file size +appender filename bytes: appender filename +file size bytes: file content +**/ +static int storage_modify_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + char appender_filename[128]; + char true_filename[128]; + char decode_buff[64]; + struct stat stat_buf; + int appender_filename_len; + int64_t nInPackLen; + int64_t file_offset; + int64_t file_bytes; + int64_t appender_file_size; + int result; + int store_path_index; + int filename_len; + int buff_len; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 3 * FDFS_PROTO_PKG_LEN_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_MODIFY_FILE, \ + pTask->client_ip, \ + nInPackLen, 3 * FDFS_PROTO_PKG_LEN_SIZE); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + appender_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + file_offset = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + file_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (appender_filename_len < FDFS_LOGIC_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH + FDFS_FILE_EXT_NAME_MAX_LEN + 1 \ + || appender_filename_len >= sizeof(appender_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid appender_filename " \ + "bytes: %d", __LINE__, \ + pTask->client_ip, appender_filename_len); + return EINVAL; + } + + if (file_offset < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "file offset: "INT64_PRINTF_FORMAT" is invalid, "\ + "which < 0", __LINE__, pTask->client_ip, file_offset); + return EINVAL; + } + + if (file_bytes < 0 || file_bytes != nInPackLen - \ + (3 * FDFS_PROTO_PKG_LEN_SIZE + appender_filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid file bytes: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, file_bytes); + return EINVAL; + } + + memcpy(appender_filename, p, appender_filename_len); + *(appender_filename + appender_filename_len) = '\0'; + p += appender_filename_len; + filename_len = appender_filename_len; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(appender_filename, \ + appender_filename_len, pClientInfo); + + if ((result=storage_split_filename_ex(appender_filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], true_filename); + if (lstat(pFileContext->filename, &stat_buf) == 0) + { + if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file: %s " \ + "is not a regular file", __LINE__, \ + pTask->client_ip, pFileContext->filename); + + return EINVAL; + } + } + else + { + result = errno != 0 ? errno : ENOENT; + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file: %s " \ + "not exist", __LINE__, \ + pTask->client_ip, pFileContext->filename); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat appednder file %s fail" \ + ", errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + result, STRERROR(result)); + } + + return result; + } + + strcpy(pFileContext->fname2log, appender_filename); + + memset(decode_buff, 0, sizeof(decode_buff)); + base64_decode_auto(&g_fdfs_base64_context, pFileContext->fname2log + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + decode_buff, &buff_len); + + appender_file_size = buff2long(decode_buff + sizeof(int) * 2); + if (!IS_APPENDER_FILE(appender_file_size)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file: %s is not a valid " \ + "appender file, file size: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, appender_filename, \ + appender_file_size); + + return EINVAL; + } + + if (file_offset > stat_buf.st_size) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file offset: "INT64_PRINTF_FORMAT \ + " is invalid, which > appender file size: " \ + OFF_PRINTF_FORMAT, __LINE__, pTask->client_ip, \ + file_offset, stat_buf.st_size); + + return EINVAL; + } + + if (file_bytes == 0) + { + return 0; + } + + pFileContext->extra_info.upload.start_time = g_current_time; + pFileContext->extra_info.upload.if_gen_filename = false; + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], true_filename); + + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_MODIFY_FILE; + pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time; + pFileContext->extra_info.upload.file_type = _FILE_TYPE_APPENDER; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + pFileContext->extra_info.upload.trunk_info.path.store_path_index = \ + store_path_index; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->open_flags = O_WRONLY | g_extra_open_file_flags; + + return storage_write_to_file(pTask, file_offset, file_bytes, \ + p - pTask->data, dio_write_file, \ + storage_modify_file_done_callback, \ + dio_modify_finish_clean_up, store_path_index); +} + +/** +8 bytes: appender filename length +8 bytes: truncated file size +appender filename bytes: appender filename +**/ +static int storage_do_truncate_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + char appender_filename[128]; + char true_filename[128]; + char decode_buff[64]; + struct stat stat_buf; + int appender_filename_len; + int64_t nInPackLen; + int64_t remain_bytes; + int64_t appender_file_size; + int result; + int store_path_index; + int filename_len; + int buff_len; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof (TrackerHeader); + + if (nInPackLen <= 2 * FDFS_PROTO_PKG_LEN_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_TRUNCATE_FILE, \ + pTask->client_ip, \ + nInPackLen, 2 * FDFS_PROTO_PKG_LEN_SIZE); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + appender_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + remain_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (appender_filename_len < FDFS_LOGIC_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH + FDFS_FILE_EXT_NAME_MAX_LEN + 1 \ + || appender_filename_len >= sizeof(appender_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid appender_filename " \ + "bytes: %d", __LINE__, \ + pTask->client_ip, appender_filename_len); + return EINVAL; + } + + if (remain_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid file bytes: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, remain_bytes); + return EINVAL; + } + + memcpy(appender_filename, p, appender_filename_len); + *(appender_filename + appender_filename_len) = '\0'; + p += appender_filename_len; + filename_len = appender_filename_len; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(appender_filename, \ + appender_filename_len, pClientInfo); + + if ((result=storage_split_filename_ex(appender_filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], true_filename); + if (lstat(pFileContext->filename, &stat_buf) == 0) + { + if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file: %s " \ + "is not a regular file", __LINE__, \ + pTask->client_ip, pFileContext->filename); + + return EINVAL; + } + } + else + { + result = errno != 0 ? errno : ENOENT; + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file: %s " \ + "not exist", __LINE__, \ + pTask->client_ip, pFileContext->filename); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat appednder file %s fail" \ + ", errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + result, STRERROR(result)); + } + + return result; + } + + strcpy(pFileContext->fname2log, appender_filename); + + memset(decode_buff, 0, sizeof(decode_buff)); + base64_decode_auto(&g_fdfs_base64_context, pFileContext->fname2log + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + decode_buff, &buff_len); + + appender_file_size = buff2long(decode_buff + sizeof(int) * 2); + if (!IS_APPENDER_FILE(appender_file_size)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file: %s is not a valid " \ + "appender file, file size: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, appender_filename, \ + appender_file_size); + + return EINVAL; + } + + if (remain_bytes == stat_buf.st_size) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, truncated file size: " \ + INT64_PRINTF_FORMAT" == appender file size: " \ + OFF_PRINTF_FORMAT", skip truncate file", \ + __LINE__, pTask->client_ip, \ + remain_bytes, stat_buf.st_size); + return 0; + } + if (remain_bytes > stat_buf.st_size) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, truncated file size: " \ + INT64_PRINTF_FORMAT" is invalid, " \ + "which > appender file size: " \ + OFF_PRINTF_FORMAT, __LINE__, pTask->client_ip, \ + remain_bytes, stat_buf.st_size); + + return EINVAL; + } + + pFileContext->extra_info.upload.start_time = g_current_time; + pFileContext->extra_info.upload.if_gen_filename = false; + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], true_filename); + + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_TRUNCATE_FILE; + pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time; + pFileContext->extra_info.upload.file_type = _FILE_TYPE_APPENDER; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + pFileContext->extra_info.upload.trunk_info.path.store_path_index = \ + store_path_index; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->open_flags = O_WRONLY | g_extra_open_file_flags; + + return storage_write_to_file(pTask, remain_bytes, \ + stat_buf.st_size, 0, dio_truncate_file, \ + storage_do_truncate_file_done_callback, \ + dio_truncate_finish_clean_up, store_path_index); +} + +/** +8 bytes: master filename length +8 bytes: file size +FDFS_FILE_PREFIX_MAX_LEN bytes : filename prefix +FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.) +master filename bytes: master filename +file size bytes: file content +**/ +static int storage_upload_slave_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + FDFSTrunkHeader trunkHeader; + char *p; + char filename[128]; + char master_filename[128]; + char true_filename[128]; + char prefix_name[FDFS_FILE_PREFIX_MAX_LEN + 1]; + char file_ext_name[FDFS_FILE_PREFIX_MAX_LEN + 1]; + char reserved_space_str[32]; + int master_filename_len; + int64_t nInPackLen; + int64_t file_bytes; + int crc32; + int result; + int store_path_index; + int filename_len; + struct stat stat_buf; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + + if (nInPackLen <= 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_FILE_PREFIX_MAX_LEN + FDFS_FILE_EXT_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, \ + nInPackLen, 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + master_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + file_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (master_filename_len <= FDFS_LOGIC_FILE_PATH_LEN || \ + master_filename_len >= sizeof(master_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid master_filename " \ + "bytes: %d", __LINE__, \ + pTask->client_ip, master_filename_len); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + if (file_bytes < 0 || file_bytes != nInPackLen - \ + (2 * FDFS_PROTO_PKG_LEN_SIZE + FDFS_FILE_PREFIX_MAX_LEN + + FDFS_FILE_EXT_NAME_MAX_LEN + master_filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid file bytes: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, file_bytes); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(prefix_name, p, FDFS_FILE_PREFIX_MAX_LEN); + *(prefix_name + FDFS_FILE_PREFIX_MAX_LEN) = '\0'; + p += FDFS_FILE_PREFIX_MAX_LEN; + if (*prefix_name == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, prefix_name is empty!", \ + __LINE__, pTask->client_ip); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + if ((result=fdfs_validate_filename(prefix_name)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, prefix_name: %s " \ + "is invalid!", __LINE__, \ + pTask->client_ip, prefix_name); + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + memcpy(file_ext_name, p, FDFS_FILE_EXT_NAME_MAX_LEN); + *(file_ext_name + FDFS_FILE_EXT_NAME_MAX_LEN) = '\0'; + p += FDFS_FILE_EXT_NAME_MAX_LEN; + if ((result=fdfs_validate_filename(file_ext_name)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file_ext_name: %s " \ + "is invalid!", __LINE__, \ + pTask->client_ip, file_ext_name); + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + memcpy(master_filename, p, master_filename_len); + *(master_filename + master_filename_len) = '\0'; + p += master_filename_len; + + filename_len = master_filename_len; + if ((result=storage_split_filename_ex(master_filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + if (!storage_check_reserved_space_path(g_path_space_list \ + [store_path_index].total_mb, g_path_space_list \ + [store_path_index].free_mb - (file_bytes / FDFS_ONE_MB), \ + g_avg_storage_reserved_mb)) + { + logError("file: "__FILE__", line: %d, " \ + "no space to upload file, " + "free space: %d MB is too small, file bytes: " \ + INT64_PRINTF_FORMAT", reserved space: %s", __LINE__,\ + g_path_space_list[store_path_index].free_mb, \ + file_bytes, fdfs_storage_reserved_space_to_string_ex(\ + g_storage_reserved_space.flag, \ + g_avg_storage_reserved_mb, \ + g_path_space_list[store_path_index].total_mb, \ + g_storage_reserved_space.rs.ratio, \ + reserved_space_str)); + pClientInfo->total_length = sizeof(TrackerHeader); + return ENOSPC; + } + + if ((result=trunk_file_lstat(store_path_index, true_filename, \ + filename_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &trunkHeader)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat logic file %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + master_filename, result, STRERROR(result)); + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + strcpy(pFileContext->extra_info.upload.file_ext_name, file_ext_name); + storage_format_ext_name(file_ext_name, \ + pFileContext->extra_info.upload.formatted_ext_name); + pFileContext->extra_info.upload.start_time = g_current_time; + pFileContext->extra_info.upload.if_gen_filename = g_check_file_duplicate; + pFileContext->extra_info.upload.if_sub_path_alloced = false; + pFileContext->extra_info.upload.trunk_info.path. \ + store_path_index = store_path_index; + if ((result=fdfs_gen_slave_filename(true_filename, \ + prefix_name, file_ext_name, filename, &filename_len)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + if (g_use_access_log) + { + snprintf(pFileContext->fname2log, \ + sizeof(pFileContext->fname2log), \ + "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + store_path_index, filename); + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], filename); + if (fileExists(pFileContext->filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, slave file: %s " \ + "already exist", __LINE__, \ + pTask->client_ip, pFileContext->filename); + pClientInfo->total_length = sizeof(TrackerHeader); + return EEXIST; + } + + crc32 = rand(); + *filename = '\0'; + filename_len = 0; + if ((result=storage_get_filename(pClientInfo, \ + pFileContext->extra_info.upload.start_time, \ + file_bytes, crc32, \ + pFileContext->extra_info.upload.formatted_ext_name, \ + filename, &filename_len, pFileContext->filename)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + if (*pFileContext->filename == '\0') + { + logWarning("file: "__FILE__", line: %d, " \ + "Can't generate uniq filename", __LINE__); + pClientInfo->total_length = sizeof(TrackerHeader); + return EBUSY; + } + + pFileContext->calc_crc32 = g_check_file_duplicate || \ + g_store_slave_file_use_link; + if (!pFileContext->calc_crc32) + { + pFileContext->crc32 = 0; + } + pFileContext->calc_file_hash = g_check_file_duplicate; + + strcpy(pFileContext->extra_info.upload.master_filename, master_filename); + strcpy(pFileContext->extra_info.upload.prefix_name, prefix_name); + + pFileContext->extra_info.upload.file_type = _FILE_TYPE_SLAVE | \ + _FILE_TYPE_REGULAR; + pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_CREATE_FILE; + pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + pFileContext->extra_info.upload.trunk_info.path.store_path_index = \ + store_path_index; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->open_flags = O_WRONLY | O_CREAT | O_TRUNC \ + | g_extra_open_file_flags; + + return storage_write_to_file(pTask, 0, file_bytes, p - pTask->data, \ + dio_write_file, storage_upload_file_done_callback, \ + dio_write_finish_clean_up, store_path_index); +} + +/** +8 bytes: filename bytes +8 bytes: file size +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +file size bytes: file content +**/ +static int storage_sync_copy_file(struct fast_task_info *pTask, \ + const char proto_cmd) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TaskDealFunc deal_func; + DisconnectCleanFunc clean_func; + FDFSTrunkHeader trunkHeader; + char *p; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char filename[128]; + int filename_len; + int64_t nInPackLen; + int64_t file_bytes; + int64_t file_offset; + int result; + int store_path_index; + bool have_file_content; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT"is not correct, " \ + "expect length > %d", __LINE__, \ + proto_cmd, pTask->client_ip, nInPackLen, \ + 2 * FDFS_PROTO_PKG_LEN_SIZE + 4+FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + file_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (filename_len < 0 || filename_len >= sizeof(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d is invalid, " \ + "which < 0 or >= %d", __LINE__, pTask->client_ip, \ + filename_len, (int)sizeof(filename)); + return EINVAL; + } + + if (file_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "file size: "INT64_PRINTF_FORMAT" is invalid, "\ + "which < 0", __LINE__, pTask->client_ip, file_bytes); + return EINVAL; + } + + pFileContext->timestamp2log = buff2int(p); + p += 4; + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", __LINE__, \ + pTask->client_ip, group_name, g_group_name); + return EINVAL; + } + + have_file_content = ((TrackerHeader *)pTask->data)->status == 0; + if (have_file_content) + { + if (file_bytes != nInPackLen - (2*FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "file size: "INT64_PRINTF_FORMAT \ + " != remain bytes: "INT64_PRINTF_FORMAT"", \ + __LINE__, pTask->client_ip, file_bytes, \ + nInPackLen - (2*FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len)); + return EINVAL; + } + } + else + { + if (0 != nInPackLen - (2*FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + " remain bytes: "INT64_PRINTF_FORMAT" != 0 ", \ + __LINE__, pTask->client_ip, \ + nInPackLen - (2*FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len)); + return EINVAL; + } + } + + memcpy(filename, p, filename_len); + *(filename + filename_len) = '\0'; + p += filename_len; + + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + if (proto_cmd == STORAGE_PROTO_CMD_SYNC_CREATE_FILE) + { + pFileContext->sync_flag = STORAGE_OP_TYPE_REPLICA_CREATE_FILE; + } + else + { + pFileContext->sync_flag = STORAGE_OP_TYPE_REPLICA_UPDATE_FILE; + } + + if (have_file_content) + { + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + } + else + { + pFileContext->op = FDFS_STORAGE_FILE_OP_DISCARD; + *(pFileContext->filename) = '\0'; + } + + pFileContext->extra_info.upload.file_type = \ + _FILE_TYPE_REGULAR; + if (proto_cmd == STORAGE_PROTO_CMD_SYNC_CREATE_FILE && \ + have_file_content) + { + struct stat stat_buf; + + if ((result=trunk_file_lstat(store_path_index, \ + true_filename, filename_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &trunkHeader)) != 0) + { + if (result != ENOENT) //accept no exist + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat logic file %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + filename, result, STRERROR(result)); + return result; + } + } + else if (!S_ISREG(stat_buf.st_mode)) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, logic file %s is not " \ + "a regular file, will be overwrited", \ + __LINE__, pTask->client_ip, filename); + } + else if (stat_buf.st_size != file_bytes) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, logic file %s, " \ + "my file size: "OFF_PRINTF_FORMAT \ + " != src file size: "INT64_PRINTF_FORMAT \ + ", will be overwrited", __LINE__, \ + pTask->client_ip, filename, \ + stat_buf.st_size, file_bytes); + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, data file: %s " \ + "already exists, ignore it", \ + __LINE__, proto_cmd, \ + pTask->client_ip, filename); + + pFileContext->op = FDFS_STORAGE_FILE_OP_DISCARD; + *(pFileContext->filename) = '\0'; + } + + if (IS_TRUNK_FILE_BY_ID(pFileContext->extra_info. \ + upload.trunk_info)) + { + pFileContext->extra_info.upload.file_type |= \ + _FILE_TYPE_TRUNK; + } + } + + if (pFileContext->op == FDFS_STORAGE_FILE_OP_WRITE) + { + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + pFileContext->crc32 = trunkHeader.crc32; + pFileContext->extra_info.upload.start_time = trunkHeader.mtime; + snprintf(pFileContext->extra_info.upload.formatted_ext_name, \ + sizeof(pFileContext->extra_info.upload.formatted_ext_name), + "%s", trunkHeader.formatted_ext_name); + + clean_func = dio_trunk_write_finish_clean_up; + file_offset = TRUNK_FILE_START_OFFSET(pFileContext-> \ + extra_info.upload.trunk_info); + trunk_get_full_filename(&pFileContext-> \ + extra_info.upload.trunk_info, pFileContext->filename, \ + sizeof(pFileContext->filename)); + pFileContext->extra_info.upload.before_open_callback = \ + dio_check_trunk_file_when_sync; + pFileContext->extra_info.upload.before_close_callback = \ + dio_write_chunk_header; + pFileContext->open_flags = O_RDWR | g_extra_open_file_flags; + } + else + { + #define MKTEMP_MAX_COUNT 10 + int i; + struct stat stat_buf; + + for (i=0; i < MKTEMP_MAX_COUNT; i++) + { + pthread_mutex_lock(&g_storage_thread_lock); + + sprintf(pFileContext->filename, "%s/data/.cp" \ + INT64_PRINTF_FORMAT".tmp", \ + g_fdfs_store_paths.paths[store_path_index], \ + temp_file_sequence++); + + pthread_mutex_unlock(&g_storage_thread_lock); + + if (stat(pFileContext->filename, &stat_buf) == 0) + { + if (g_current_time - stat_buf.st_mtime > 600) + { + if (unlink(pFileContext->filename) != 0 + && errno != ENOENT) + { + logWarning("file: "__FILE__", line: %d"\ + ", client ip: %s, unlink temp file %s "\ + " fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, \ + errno, STRERROR(errno)); + continue; + } + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, temp file %s already "\ + "exists", __LINE__, pTask->client_ip, \ + pFileContext->filename); + continue; + } + } + + break; + } + + if (i == MKTEMP_MAX_COUNT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, make temp file fail", \ + __LINE__, pTask->client_ip); + return EAGAIN; + } + + clean_func = dio_write_finish_clean_up; + file_offset = 0; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + pFileContext->open_flags = O_WRONLY | O_CREAT | O_TRUNC \ + | g_extra_open_file_flags; + } + + deal_func = dio_write_file; + } + else + { + file_offset = 0; + deal_func = dio_discard_file; + clean_func = NULL; + + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + } + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + strcpy(pFileContext->fname2log, filename); + + if (have_file_content) + { + return storage_write_to_file(pTask, file_offset, file_bytes, \ + p - pTask->data, deal_func, \ + storage_sync_copy_file_done_callback, \ + clean_func, store_path_index); + } + else + { + storage_sync_copy_file_done_callback(pTask, 0); + return STORAGE_STATUE_DEAL_FILE; + } +} + +/** +8 bytes: filename bytes +8 bytes: start offset +8 bytes: append bytes +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +file size bytes: file content +**/ +static int storage_sync_append_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TaskDealFunc deal_func; + char *p; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char filename[128]; + bool need_write_file; + int filename_len; + int64_t nInPackLen; + int64_t start_offset; + int64_t append_bytes; + struct stat stat_buf; + int result; + int store_path_index; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT"is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_SYNC_APPEND_FILE, \ + pTask->client_ip, nInPackLen, \ + 3 * FDFS_PROTO_PKG_LEN_SIZE + 4+FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + start_offset = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + append_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (filename_len < 0 || filename_len >= sizeof(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d is invalid, " \ + "which < 0 or >= %d", __LINE__, pTask->client_ip, \ + filename_len, (int)sizeof(filename)); + return EINVAL; + } + + if (start_offset < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "start offset: "INT64_PRINTF_FORMAT" is invalid, "\ + "which < 0", __LINE__, pTask->client_ip, start_offset); + return EINVAL; + } + + if (append_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "append bytes: "INT64_PRINTF_FORMAT" is invalid, "\ + "which < 0", __LINE__, pTask->client_ip, append_bytes); + return EINVAL; + } + + pFileContext->timestamp2log = buff2int(p); + p += 4; + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", __LINE__, \ + pTask->client_ip, group_name, g_group_name); + return EINVAL; + } + + if (append_bytes != nInPackLen - (3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "file size: "INT64_PRINTF_FORMAT \ + " != remain bytes: "INT64_PRINTF_FORMAT"", \ + __LINE__, pTask->client_ip, append_bytes, \ + nInPackLen - (3 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len)); + return EINVAL; + } + + memcpy(filename, p, filename_len); + *(filename + filename_len) = '\0'; + p += filename_len; + + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + + if (lstat(pFileContext->filename, &stat_buf) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat file %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, result, STRERROR(result)); + return result; + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file %s not exists, " \ + "will be resynced", __LINE__, pTask->client_ip, \ + pFileContext->filename); + need_write_file = false; + } + } + else if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s is not a regular " \ + "file, will be ignored", __LINE__, \ + pTask->client_ip, pFileContext->filename); + need_write_file = false; + } + else if (stat_buf.st_size == start_offset) + { + need_write_file = true; + } + else if (stat_buf.st_size > start_offset) + { + if (stat_buf.st_size >= start_offset + append_bytes) + { + logDebug("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s, my file size: " \ + OFF_PRINTF_FORMAT" >= src file size: " \ + INT64_PRINTF_FORMAT", do not append", \ + __LINE__, pTask->client_ip, pFileContext->filename, \ + stat_buf.st_size, start_offset + append_bytes); + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s, my file size: " \ + OFF_PRINTF_FORMAT" > "INT64_PRINTF_FORMAT \ + ", but < "INT64_PRINTF_FORMAT", need be resynced", \ + __LINE__, pTask->client_ip, pFileContext->filename, \ + stat_buf.st_size, start_offset, \ + start_offset + append_bytes); + + } + + need_write_file = false; + } + else //stat_buf.st_size < start_offset + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s, my file size: " \ + OFF_PRINTF_FORMAT" < start offset " \ + INT64_PRINTF_FORMAT", need to resync this file!", \ + __LINE__, pTask->client_ip, pFileContext->filename, \ + stat_buf.st_size, start_offset); + need_write_file = false; + } + + pFileContext->sync_flag = STORAGE_OP_TYPE_REPLICA_APPEND_FILE; + + if (need_write_file) + { + deal_func = dio_write_file; + pFileContext->op = FDFS_STORAGE_FILE_OP_APPEND; + pFileContext->open_flags = O_WRONLY | O_APPEND | g_extra_open_file_flags; + + snprintf(pFileContext->fname2log, \ + sizeof(pFileContext->fname2log), \ + "%s "INT64_PRINTF_FORMAT" "INT64_PRINTF_FORMAT, \ + filename, start_offset, append_bytes); + } + else + { + deal_func = dio_discard_file; + pFileContext->op = FDFS_STORAGE_FILE_OP_DISCARD; + pFileContext->open_flags = 0; + } + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + + return storage_write_to_file(pTask, start_offset, append_bytes, \ + p - pTask->data, deal_func, \ + storage_sync_modify_file_done_callback, \ + dio_append_finish_clean_up, store_path_index); +} + +/** +8 bytes: filename bytes +8 bytes: start offset +8 bytes: append bytes +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +file size bytes: file content +**/ +static int storage_sync_modify_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TaskDealFunc deal_func; + char *p; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char filename[128]; + bool need_write_file; + int filename_len; + int64_t nInPackLen; + int64_t start_offset; + int64_t modify_bytes; + struct stat stat_buf; + int result; + int store_path_index; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT"is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_SYNC_MODIFY_FILE, \ + pTask->client_ip, nInPackLen, \ + 3 * FDFS_PROTO_PKG_LEN_SIZE + 4 + \ + FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + start_offset = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + modify_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (filename_len < 0 || filename_len >= sizeof(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d is invalid, " \ + "which < 0 or >= %d", __LINE__, pTask->client_ip, \ + filename_len, (int)sizeof(filename)); + return EINVAL; + } + + if (start_offset < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "start offset: "INT64_PRINTF_FORMAT" is invalid, "\ + "which < 0", __LINE__, pTask->client_ip, start_offset); + return EINVAL; + } + + if (modify_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "modify file bytes: "INT64_PRINTF_FORMAT" is invalid, "\ + "which < 0", __LINE__, pTask->client_ip, modify_bytes); + return EINVAL; + } + + pFileContext->timestamp2log = buff2int(p); + p += 4; + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", __LINE__, \ + pTask->client_ip, group_name, g_group_name); + return EINVAL; + } + + if (modify_bytes != nInPackLen - (3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "file size: "INT64_PRINTF_FORMAT \ + " != remain bytes: "INT64_PRINTF_FORMAT"", \ + __LINE__, pTask->client_ip, modify_bytes, \ + nInPackLen - (3 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + filename_len)); + return EINVAL; + } + + memcpy(filename, p, filename_len); + *(filename + filename_len) = '\0'; + p += filename_len; + + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + + if (lstat(pFileContext->filename, &stat_buf) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat file %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, result, STRERROR(result)); + return result; + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file %s not exists, " \ + "will be resynced", __LINE__, pTask->client_ip, \ + pFileContext->filename); + need_write_file = false; + } + } + else if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s is not a regular " \ + "file, will be ignored", __LINE__, \ + pTask->client_ip, pFileContext->filename); + need_write_file = false; + } + else if (stat_buf.st_size < start_offset) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s, my file size: " \ + OFF_PRINTF_FORMAT" < start offset " \ + INT64_PRINTF_FORMAT", need to resync this file!", \ + __LINE__, pTask->client_ip, pFileContext->filename, \ + stat_buf.st_size, start_offset); + need_write_file = false; + } + else + { + need_write_file = true; + } + + pFileContext->sync_flag = STORAGE_OP_TYPE_REPLICA_MODIFY_FILE; + if (need_write_file) + { + deal_func = dio_write_file; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->open_flags = O_WRONLY | g_extra_open_file_flags; + + snprintf(pFileContext->fname2log, \ + sizeof(pFileContext->fname2log), \ + "%s "INT64_PRINTF_FORMAT" "INT64_PRINTF_FORMAT, \ + filename, start_offset, modify_bytes); + } + else + { + deal_func = dio_discard_file; + pFileContext->op = FDFS_STORAGE_FILE_OP_DISCARD; + pFileContext->open_flags = 0; + } + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + + return storage_write_to_file(pTask, start_offset, modify_bytes, \ + p - pTask->data, deal_func, \ + storage_sync_modify_file_done_callback, \ + dio_modify_finish_clean_up, store_path_index); +} + +/** +8 bytes: filename bytes +8 bytes: old file size +8 bytes: new file size +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +**/ +static int storage_sync_truncate_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char filename[128]; + int filename_len; + int64_t nInPackLen; + int64_t old_file_size; + int64_t new_file_size; + struct stat stat_buf; + int result; + int store_path_index; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT"is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_SYNC_TRUNCATE_FILE, \ + pTask->client_ip, nInPackLen, \ + 3 * FDFS_PROTO_PKG_LEN_SIZE + 4 + + FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + old_file_size = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + new_file_size = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (filename_len < 0 || filename_len >= sizeof(filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d is invalid, " \ + "which < 0 or >= %d", __LINE__, \ + pTask->client_ip, filename_len, \ + (int)sizeof(filename)); + return EINVAL; + } + + if (filename_len != nInPackLen - (3 * FDFS_PROTO_PKG_LEN_SIZE +\ + 4 + FDFS_GROUP_NAME_MAX_LEN)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d != %d", __LINE__, \ + pTask->client_ip, filename_len, \ + (int)nInPackLen - (3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN)); + return EINVAL; + } + + if (old_file_size < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "start offset: "INT64_PRINTF_FORMAT \ + "is invalid, which < 0", __LINE__, \ + pTask->client_ip, old_file_size); + return EINVAL; + } + + if (new_file_size < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "modify file bytes: "INT64_PRINTF_FORMAT \ + " is invalid, which < 0", __LINE__, \ + pTask->client_ip, new_file_size); + return EINVAL; + } + + pFileContext->timestamp2log = buff2int(p); + p += 4; + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", __LINE__, \ + pTask->client_ip, group_name, g_group_name); + return EINVAL; + } + + memcpy(filename, p, filename_len); + *(filename + filename_len) = '\0'; + p += filename_len; + + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], \ + true_filename); + + if (lstat(pFileContext->filename, &stat_buf) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, stat file %s fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename, result, STRERROR(result)); + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, appender file %s not exists, " \ + "will be resynced", __LINE__, pTask->client_ip, \ + pFileContext->filename); + } + return result; + } + if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s is not a regular " \ + "file, will be ignored", __LINE__, \ + pTask->client_ip, pFileContext->filename); + return EEXIST; + } + if (stat_buf.st_size != old_file_size) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s, my file size: " \ + OFF_PRINTF_FORMAT" != before truncated size: " \ + INT64_PRINTF_FORMAT", skip!", __LINE__, \ + pTask->client_ip, pFileContext->filename, \ + stat_buf.st_size, old_file_size); + return EEXIST; + } + + pFileContext->sync_flag = STORAGE_OP_TYPE_REPLICA_TRUNCATE_FILE; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->open_flags = O_WRONLY | g_extra_open_file_flags; + + snprintf(pFileContext->fname2log, \ + sizeof(pFileContext->fname2log), \ + "%s "INT64_PRINTF_FORMAT" "INT64_PRINTF_FORMAT, \ + filename, old_file_size, new_file_size); + + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + pFileContext->extra_info.upload.before_open_callback = NULL; + pFileContext->extra_info.upload.before_close_callback = NULL; + + return storage_write_to_file(pTask, new_file_size, \ + old_file_size, 0, dio_truncate_file, \ + storage_sync_truncate_file_done_callback, \ + dio_truncate_finish_clean_up, store_path_index); +} + +/** +8 bytes: dest(link) filename length +8 bytes: source filename length +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +dest filename length: dest filename +source filename length: source filename +**/ +static int storage_do_sync_link_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + char *p; + struct stat stat_buf; + FDFSTrunkHeader srcTrunkHeader; + FDFSTrunkHeader destTrunkHeader; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char dest_filename[128]; + char dest_true_filename[128]; + char src_filename[128]; + char src_true_filename[128]; + char src_full_filename[MAX_PATH_SIZE]; + char binlog_buff[256]; + bool need_create_link; + int64_t nInPackLen; + int dest_filename_len; + int dest_true_filename_len; + int src_filename_len; + int src_true_filename_len; + int result; + int dest_store_path_index; + int src_store_path_index; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + do + { + p = pTask->data + sizeof(TrackerHeader); + + dest_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + src_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE + 4; + + if (src_filename_len < 0 || src_filename_len >= sizeof(src_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d is invalid, " \ + "which < 0 or >= %d", \ + __LINE__, pTask->client_ip, \ + src_filename_len, (int)sizeof(src_filename)); + result = EINVAL; + break; + } + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + result = EINVAL; + break; + } + + if (nInPackLen != 2 * FDFS_PROTO_PKG_LEN_SIZE + 4 + \ + FDFS_GROUP_NAME_MAX_LEN + dest_filename_len + src_filename_len) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "pgk length: "INT64_PRINTF_FORMAT \ + " != bytes: %d", __LINE__, pTask->client_ip, \ + nInPackLen, 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + dest_filename_len + \ + src_filename_len); + result = EINVAL; + break; + } + + memcpy(dest_filename, p, dest_filename_len); + *(dest_filename + dest_filename_len) = '\0'; + p += dest_filename_len; + + memcpy(src_filename, p, src_filename_len); + *(src_filename + src_filename_len) = '\0'; + p += src_filename_len; + + dest_true_filename_len = dest_filename_len; + if ((result=storage_split_filename_ex(dest_filename, \ + &dest_true_filename_len, dest_true_filename, \ + &dest_store_path_index)) != 0) + { + break; + } + + src_true_filename_len = src_filename_len; + if ((result=storage_split_filename_ex(src_filename, \ + &src_true_filename_len, src_true_filename, \ + &src_store_path_index)) != 0) + { + break; + } + if ((result=fdfs_check_data_filename(src_true_filename, \ + src_filename_len)) != 0) + { + break; + } + + memset(&destTrunkHeader, 0, sizeof(destTrunkHeader)); + if (trunk_file_lstat(dest_store_path_index, dest_true_filename, \ + dest_true_filename_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &destTrunkHeader) == 0) + { + need_create_link = false; + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, logic link file: %s " \ + "already exists, ignore it", __LINE__, \ + pTask->client_ip, dest_filename); + } + else + { + FDFSTrunkFullInfo trunkInfo; + if (trunk_file_lstat(src_store_path_index, src_true_filename, \ + src_true_filename_len, &stat_buf, \ + &trunkInfo, &srcTrunkHeader) != 0) + { + need_create_link = false; + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, logic source file: %s " \ + "not exists, ignore it", __LINE__, \ + pTask->client_ip, src_filename); + } + else + { + need_create_link = true; + } + } + + if (need_create_link) + { + if (IS_TRUNK_FILE_BY_ID(pFileContext->extra_info.upload.trunk_info)) + { + pFileContext->extra_info.upload.file_type = \ + _FILE_TYPE_LINK; + pFileContext->extra_info.upload.start_time = \ + destTrunkHeader.mtime; + pFileContext->crc32 = destTrunkHeader.crc32; + strcpy(pFileContext->extra_info.upload. \ + formatted_ext_name, \ + destTrunkHeader.formatted_ext_name); + + pTask->length = pTask->size; + p = pTask->data + (pTask->length - src_filename_len); + if (p < pTask->data + sizeof(TrackerHeader)) + { + logError("file: "__FILE__", line: %d, " \ + "task buffer size: %d is too small", \ + __LINE__, pTask->size); + break; + } + + memcpy(p, src_filename, src_filename_len); + result = storage_trunk_do_create_link(pTask, \ + src_filename_len, p - pTask->data, \ + dio_check_trunk_file_when_sync, NULL); + if (result != 0) + { + break; + } + } + else + { + snprintf(pFileContext->filename, sizeof(pFileContext->filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[dest_store_path_index], \ + dest_true_filename); + + snprintf(src_full_filename, sizeof(src_full_filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[src_store_path_index], \ + src_true_filename); + if (symlink(src_full_filename, pFileContext->filename) != 0) + { + result = errno != 0 ? errno : EPERM; + if (result == EEXIST) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip: %s, data file: %s " \ + "already exists, ignore it", __LINE__, \ + pTask->client_ip, pFileContext->filename); + result = 0; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, link file %s to %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + src_full_filename, pFileContext->filename, \ + result, STRERROR(result)); + break; + } + + } + } + } + + snprintf(binlog_buff, sizeof(binlog_buff), "%s %s", \ + dest_filename, src_filename); + result = storage_binlog_write(pFileContext->timestamp2log, \ + STORAGE_OP_TYPE_REPLICA_CREATE_LINK, \ + binlog_buff); + } while (0); + + CHECK_AND_WRITE_TO_STAT_FILE1 + + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); + + return result; +} + +static int storage_sync_link_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + char dest_filename[128]; + char dest_true_filename[128]; + int64_t nInPackLen; + int dest_filename_len; + int src_filename_len; + int result; + int dest_store_path_index; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + + if (nInPackLen <= 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + pTask->client_ip, nInPackLen, \ + 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + dest_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + src_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + if (dest_filename_len < 0 || dest_filename_len >= sizeof(dest_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "filename length: %d is invalid, " \ + "which < 0 or >= %d", \ + __LINE__, pTask->client_ip, \ + dest_filename_len, (int)sizeof(dest_filename)); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + pFileContext->timestamp2log = buff2int(p); + p += 4 + FDFS_GROUP_NAME_MAX_LEN; + if (nInPackLen != 2 * FDFS_PROTO_PKG_LEN_SIZE + 4 + \ + FDFS_GROUP_NAME_MAX_LEN + dest_filename_len + src_filename_len) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, in request pkg, " \ + "pgk length: "INT64_PRINTF_FORMAT \ + " != bytes: %d", __LINE__, pTask->client_ip, \ + nInPackLen, 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + dest_filename_len + \ + src_filename_len); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(dest_filename, p, dest_filename_len); + *(dest_filename + dest_filename_len) = '\0'; + p += dest_filename_len; + + if ((result=storage_split_filename_ex(dest_filename, \ + &dest_filename_len, dest_true_filename, \ + &dest_store_path_index)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + if ((result=fdfs_check_data_filename(dest_true_filename, \ + dest_filename_len)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + pFileContext->extra_info.upload.trunk_info.path.store_path_index = \ + dest_store_path_index; + + pClientInfo->deal_func = storage_do_sync_link_file; + + pFileContext->fd = -1; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, dest_store_path_index, pFileContext->op); + + if ((result=storage_dio_queue_push(pTask)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + return STORAGE_STATUE_DEAL_FILE; +} + +/** +pkg format: +Header +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +**/ +static int storage_server_get_metadata(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + int result; + int store_path_index; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + struct stat stat_buf; + FDFSTrunkFullInfo trunkInfo; + FDFSTrunkHeader trunkHeader; + char *filename; + int filename_len; + int64_t file_bytes; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, nInPackLen, FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + if (nInPackLen >= pTask->size) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is too large, " \ + "expect length should < %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, \ + nInPackLen, pTask->size); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + filename = p; + filename_len = nInPackLen - FDFS_GROUP_NAME_MAX_LEN; + *(filename + filename_len) = '\0'; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(filename, filename_len, \ + pClientInfo); + + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, \ + filename_len)) != 0) + { + return result; + } + + if ((result=trunk_file_stat(store_path_index, \ + true_filename, filename_len, &stat_buf, \ + &trunkInfo, &trunkHeader)) != 0) + { + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "logic", filename) + return result; + } + + sprintf(pFileContext->filename, "%s/data/%s%s", \ + g_fdfs_store_paths.paths[store_path_index], \ + true_filename, FDFS_STORAGE_META_FILE_EXT); + if (lstat(pFileContext->filename, &stat_buf) == 0) + { + if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "%s is not a regular file", \ + __LINE__, pFileContext->filename); + return EISDIR; + } + + file_bytes = stat_buf.st_size; + } + else + { + result = errno != 0 ? errno : ENOENT; + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "regular", pFileContext->filename) + return result; + } + + pFileContext->fd = -1; + return storage_read_from_file(pTask, 0, file_bytes, \ + storage_get_metadata_done_callback, store_path_index); +} + +/** +pkg format: +Header +8 bytes: file offset +8 bytes: download file bytes +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +**/ +static int storage_server_download_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + int result; + int store_path_index; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char *filename; + int filename_len; + int64_t file_offset; + int64_t download_bytes; + int64_t file_bytes; + struct stat stat_buf; + int64_t nInPackLen; + FDFSTrunkFullInfo trunkInfo; + FDFSTrunkHeader trunkHeader; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 16 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, \ + nInPackLen, 16 + FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + if (nInPackLen >= pTask->size) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is too large, " \ + "expect length should < %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, \ + nInPackLen, pTask->size); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + + file_offset = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + download_bytes = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (file_offset < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid file offset: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + pTask->client_ip, file_offset); + return EINVAL; + } + if (download_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid download file bytes: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + pTask->client_ip, download_bytes); + return EINVAL; + } + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + filename = p; + filename_len = nInPackLen - (16 + FDFS_GROUP_NAME_MAX_LEN); + *(filename + filename_len) = '\0'; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(filename, filename_len, \ + pClientInfo); + + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + pFileContext->fd = -1; + result = trunk_file_stat_ex(store_path_index, \ + true_filename, filename_len, &stat_buf, \ + &trunkInfo, &trunkHeader, &pFileContext->fd); + if (IS_TRUNK_FILE_BY_ID(trunkInfo)) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_file_open_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + } + if (result == 0) + { + if (!S_ISREG(stat_buf.st_mode)) + { + logError("file: "__FILE__", line: %d, " \ + "logic file %s is not a regular file", \ + __LINE__, filename); + return EISDIR; + } + + file_bytes = stat_buf.st_size; + } + else + { + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "logic", filename) + file_bytes = 0; + return result; + } + + if (IS_TRUNK_FILE_BY_ID(trunkInfo)) + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.success_file_open_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + if (download_bytes == 0) + { + download_bytes = file_bytes - file_offset; + } + else if (download_bytes > file_bytes - file_offset) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid download file bytes: " \ + INT64_PRINTF_FORMAT" > file remain bytes: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + pTask->client_ip, download_bytes, \ + file_bytes - file_offset); + if (pFileContext->fd >= 0) + { + close(pFileContext->fd); + } + return EINVAL; + } + + if (IS_TRUNK_FILE_BY_ID(trunkInfo)) + { + trunk_get_full_filename((&trunkInfo), pFileContext->filename, \ + sizeof(pFileContext->filename)); + file_offset += TRUNK_FILE_START_OFFSET(trunkInfo); + } + else + { + sprintf(pFileContext->filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], true_filename); + } + + return storage_read_from_file(pTask, file_offset, download_bytes, \ + storage_download_file_done_callback, store_path_index); +} + +static int storage_do_delete_file(struct fast_task_info *pTask, \ + DeleteFileLogCallback log_callback, \ + FileDealDoneCallback done_callback, \ + const int store_path_index) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + pFileContext->fd = -1; + pFileContext->op = FDFS_STORAGE_FILE_OP_DELETE; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, store_path_index, pFileContext->op); + pFileContext->log_callback = log_callback; + pFileContext->done_callback = done_callback; + + if ((result=storage_dio_queue_push(pTask)) != 0) + { + return result; + } + + return STORAGE_STATUE_DEAL_FILE; +} + +static int storage_read_from_file(struct fast_task_info *pTask, \ + const int64_t file_offset, const int64_t download_bytes, \ + FileDealDoneCallback done_callback, \ + const int store_path_index) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + TrackerHeader *pHeader; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + pClientInfo->deal_func = dio_read_file; + pClientInfo->clean_func = dio_read_finish_clean_up; + pClientInfo->total_length = sizeof(TrackerHeader) + download_bytes; + pClientInfo->total_offset = 0; + + pFileContext->op = FDFS_STORAGE_FILE_OP_READ; + pFileContext->open_flags = O_RDONLY | g_extra_open_file_flags; + pFileContext->offset = file_offset; + pFileContext->start = file_offset; + pFileContext->end = file_offset + download_bytes; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, store_path_index, pFileContext->op); + pFileContext->done_callback = done_callback; + + pTask->length = sizeof(TrackerHeader); + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = 0; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(download_bytes, pHeader->pkg_len); + + if ((result=storage_dio_queue_push(pTask)) != 0) + { + if (pFileContext->fd >= 0) + { + close(pFileContext->fd); + } + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + return STORAGE_STATUE_DEAL_FILE; +} + +static int storage_write_to_file(struct fast_task_info *pTask, \ + const int64_t file_offset, const int64_t upload_bytes, \ + const int buff_offset, TaskDealFunc deal_func, \ + FileDealDoneCallback done_callback, \ + DisconnectCleanFunc clean_func, const int store_path_index) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + pClientInfo->deal_func = deal_func; + pClientInfo->clean_func = clean_func; + + pFileContext->fd = -1; + pFileContext->buff_offset = buff_offset; + pFileContext->offset = file_offset; + pFileContext->start = file_offset; + pFileContext->end = file_offset + upload_bytes; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, store_path_index, pFileContext->op); + pFileContext->done_callback = done_callback; + + if (pFileContext->calc_crc32) + { + pFileContext->crc32 = CRC32_XINIT; + } + + if (pFileContext->calc_file_hash) + { + if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH) + { + INIT_HASH_CODES4(pFileContext->file_hash_codes) + } + else + { + my_md5_init(&pFileContext->md5_context); + } + } + + if ((result=storage_dio_queue_push(pTask)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + return STORAGE_STATUE_DEAL_FILE; +} + +/** +pkg format: +Header +4 bytes: source delete timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +**/ +static int storage_sync_delete_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + FDFSTrunkHeader trunkHeader; + struct stat stat_buf; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char *filename; + int filename_len; + int result; + int store_path_index; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + if (nInPackLen <= 4 + FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length <= %d", __LINE__, \ + STORAGE_PROTO_CMD_SYNC_DELETE_FILE, \ + pTask->client_ip, \ + nInPackLen, 4 + FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + if (nInPackLen >= pTask->size) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is too large, " \ + "expect length should < %d", __LINE__, \ + STORAGE_PROTO_CMD_SYNC_DELETE_FILE, \ + pTask->client_ip, nInPackLen, pTask->size); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + pFileContext->timestamp2log = buff2int(p); + p += 4; + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + filename = p; + filename_len = nInPackLen - (4 + FDFS_GROUP_NAME_MAX_LEN); + *(filename + filename_len) = '\0'; + if ((result=storage_split_filename_ex(filename, \ + &filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0) + { + return result; + } + + if ((result=trunk_file_lstat(store_path_index, true_filename, \ + filename_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &trunkHeader)) != 0) + { + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "logic", filename) + return result; + } + if (S_ISREG(stat_buf.st_mode)) + { + pFileContext->delete_flag = STORAGE_DELETE_FLAG_FILE; + } + else if (S_ISLNK(stat_buf.st_mode)) + { + pFileContext->delete_flag = STORAGE_DELETE_FLAG_LINK; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, logic file %s is NOT a file", \ + __LINE__, pTask->client_ip, filename); + return EINVAL; + } + + if (IS_TRUNK_FILE_BY_ID(pFileContext->extra_info.upload.trunk_info)) + { + pClientInfo->deal_func = dio_delete_trunk_file; + trunk_get_full_filename((&pFileContext->extra_info.upload.\ + trunk_info), pFileContext->filename, \ + sizeof(pFileContext->filename)); + } + else + { + pClientInfo->deal_func = dio_delete_normal_file; + sprintf(pFileContext->filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], true_filename); + } + + strcpy(pFileContext->fname2log, filename); + pFileContext->sync_flag = STORAGE_OP_TYPE_REPLICA_DELETE_FILE; + return storage_do_delete_file(pTask, storage_sync_delete_file_log_error, \ + storage_sync_delete_file_done_callback, store_path_index); +} + +/** +pkg format: +Header +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename +**/ +static int storage_server_delete_file(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + FDFSTrunkHeader trunkHeader; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char true_filename[128]; + char *filename; + int filename_len; + int true_filename_len; + struct stat stat_buf; + int result; + int store_path_index; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + pFileContext->delete_flag = STORAGE_DELETE_FLAG_NONE; + if (nInPackLen <= FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length <= %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, \ + nInPackLen, FDFS_GROUP_NAME_MAX_LEN); + return EINVAL; + } + + if (nInPackLen >= pTask->size) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is too large, " \ + "expect length should < %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, \ + pTask->client_ip, nInPackLen, pTask->size); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + return EINVAL; + } + + filename = p; + filename_len = nInPackLen - FDFS_GROUP_NAME_MAX_LEN; + *(filename + filename_len) = '\0'; + + STORAGE_ACCESS_STRCPY_FNAME2LOG(filename, filename_len, \ + pClientInfo); + + true_filename_len = filename_len; + if ((result=storage_split_filename_ex(filename, \ + &true_filename_len, true_filename, &store_path_index)) != 0) + { + return result; + } + if ((result=fdfs_check_data_filename(true_filename, \ + true_filename_len)) != 0) + { + return result; + } + + if ((result=trunk_file_lstat(store_path_index, true_filename, \ + true_filename_len, &stat_buf, \ + &(pFileContext->extra_info.upload.trunk_info), \ + &trunkHeader)) != 0) + { + STORAGE_STAT_FILE_FAIL_LOG(result, pTask->client_ip, + "logic", filename) + return result; + } + if (S_ISREG(stat_buf.st_mode)) + { + pFileContext->extra_info.upload.file_type = \ + _FILE_TYPE_REGULAR; + pFileContext->delete_flag |= STORAGE_DELETE_FLAG_FILE; + } + else if (S_ISLNK(stat_buf.st_mode)) + { + pFileContext->extra_info.upload.file_type = _FILE_TYPE_LINK; + pFileContext->delete_flag |= STORAGE_DELETE_FLAG_LINK; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, file %s is NOT a file", \ + __LINE__, pTask->client_ip, \ + pFileContext->filename); + return EINVAL; + } + + if (IS_TRUNK_FILE_BY_ID(pFileContext->extra_info.upload.trunk_info)) + { + pFileContext->extra_info.upload.file_type |= _FILE_TYPE_TRUNK; + pClientInfo->deal_func = dio_delete_trunk_file; + trunk_get_full_filename((&pFileContext->extra_info.upload.\ + trunk_info), pFileContext->filename, \ + sizeof(pFileContext->filename)); + } + else + { + pClientInfo->deal_func = dio_delete_normal_file; + sprintf(pFileContext->filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], true_filename); + } + + if ((pFileContext->extra_info.upload.file_type & _FILE_TYPE_LINK) && \ + storage_is_slave_file(filename, filename_len)) + { + char full_filename[MAX_PATH_SIZE + 128]; + char src_filename[MAX_PATH_SIZE + 128]; + char src_fname2log[128]; + char *src_true_filename; + int src_filename_len; + int base_path_len; + int src_store_path_index; + int i; + + sprintf(full_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[store_path_index], true_filename); + do + { + if ((src_filename_len=readlink(full_filename, \ + src_filename, sizeof(src_filename))) < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, call readlink file %s " \ + "fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + true_filename, result, STRERROR(result)); + return result; + } + + *(src_filename + src_filename_len) = '\0'; + if (unlink(src_filename) != 0) + { + result = errno != 0 ? errno : ENOENT; + logWarning("file: "__FILE__", line: %d, " \ + "client ip:%s, unlink file %s " \ + "fail, errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + src_filename, result, STRERROR(result)); + if (result == ENOENT) + { + break; + } + return result; + } + + base_path_len = strlen(g_fdfs_store_paths.paths \ + [store_path_index]); + if (src_filename_len > base_path_len && memcmp( \ + src_filename, g_fdfs_store_paths.paths \ + [store_path_index], base_path_len) == 0) + { + src_store_path_index = store_path_index; + } + else + { + src_store_path_index = -1; + for (i=0; i base_path_len && \ + memcmp(src_filename, g_fdfs_store_paths.paths\ + [i], base_path_len) == 0) + { + src_store_path_index = i; + break; + } + } + if (src_store_path_index < 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "client ip:%s, can't get store base " \ + "path of file %s", __LINE__, \ + pTask->client_ip, src_filename); + break; + } + } + + src_true_filename = src_filename + (base_path_len + \ + (sizeof("/data/") -1)); + snprintf(src_fname2log, sizeof(src_fname2log), \ + "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + src_store_path_index, src_true_filename); + storage_binlog_write(g_current_time, \ + STORAGE_OP_TYPE_SOURCE_DELETE_FILE, \ + src_fname2log); + } while (0); + } + + strcpy(pFileContext->fname2log, filename); + return storage_do_delete_file(pTask, storage_delete_file_log_error, \ + storage_delete_fdfs_file_done_callback, \ + store_path_index); +} + +static int storage_create_link_core(struct fast_task_info *pTask, \ + SourceFileInfo *pSourceFileInfo, \ + const char *src_filename, const char *master_filename, \ + const int master_filename_len, \ + const char *prefix_name, const char *file_ext_name, \ + char *filename, int *filename_len, const bool bNeedReponse) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + FDFSTrunkFullInfo *pTrunkInfo; + int result; + struct stat stat_buf; + char true_filename[128]; + char full_filename[MAX_PATH_SIZE]; + char binlog_buff[256]; + int store_path_index; + FDFSTrunkHeader trunk_header; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + store_path_index = pFileContext->extra_info. + upload.trunk_info.path.store_path_index; + + do + { + pTrunkInfo = &(pFileContext->extra_info.upload.trunk_info); + result = trunk_file_lstat(store_path_index, \ + pSourceFileInfo->src_true_filename, \ + strlen(pSourceFileInfo->src_true_filename), \ + &stat_buf, pTrunkInfo, &trunk_header); + if (result != 0 || !S_ISREG(stat_buf.st_mode)) + { + FDHTKeyInfo key_info; + GroupArray *pGroupArray; + + result = result != 0 ? result : EINVAL; + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, logic file: %s call stat fail "\ + "or it is not a regular file, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + src_filename, result, STRERROR(result)); + if (g_check_file_duplicate) + { + pGroupArray=&((g_nio_thread_data+pClientInfo->nio_thread_index)\ + ->group_array); + //clean invalid entry + memset(&key_info, 0, sizeof(key_info)); + key_info.namespace_len = g_namespace_len; + memcpy(key_info.szNameSpace, g_key_namespace, \ + g_namespace_len); + + key_info.obj_id_len = pSourceFileInfo->src_file_sig_len; + memcpy(key_info.szObjectId, pSourceFileInfo->src_file_sig, + key_info.obj_id_len); + key_info.key_len = sizeof(FDHT_KEY_NAME_FILE_ID) - 1; + memcpy(key_info.szKey, FDHT_KEY_NAME_FILE_ID, \ + sizeof(FDHT_KEY_NAME_FILE_ID) - 1); + fdht_delete_ex(pGroupArray, g_keep_alive, &key_info); + + key_info.obj_id_len = snprintf(key_info.szObjectId, \ + sizeof(key_info.szObjectId), "%s/%s", \ + g_group_name, src_filename); + key_info.key_len = sizeof(FDHT_KEY_NAME_REF_COUNT) - 1; + memcpy(key_info.szKey, FDHT_KEY_NAME_REF_COUNT, \ + key_info.key_len); + fdht_delete_ex(pGroupArray, g_keep_alive, &key_info); + } + + break; + } + + if (master_filename_len == 0 && IS_TRUNK_FILE_BY_ID((*pTrunkInfo))) + { + if (!g_if_use_trunk_file) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid trunked src file: %s, "\ + "because i don't support trunked file!", \ + __LINE__, pTask->client_ip, src_filename); + result = EINVAL; + break; + } + + pFileContext->extra_info.upload.file_type |= _FILE_TYPE_TRUNK; + } + + if (master_filename_len > 0) + { + int master_store_path_index; + + *filename_len = master_filename_len; + if ((result=storage_split_filename_ex( \ + master_filename, filename_len, true_filename, \ + &master_store_path_index)) != 0) + { + break; + } + + if (master_store_path_index != store_path_index) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid master store " \ + "path index: %d != source store path " \ + "index: %d", __LINE__, pTask->client_ip, \ + master_store_path_index, store_path_index); + result = EINVAL; + break; + } + + if ((result=fdfs_check_data_filename(true_filename, \ + *filename_len)) != 0) + { + break; + } + + if ((result=fdfs_gen_slave_filename( \ + true_filename, prefix_name, file_ext_name, \ + filename, filename_len)) != 0) + { + break; + } + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[store_path_index], \ + filename); + if (fileExists(full_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, slave file: %s " \ + "already exist", __LINE__, \ + pTask->client_ip, full_filename); + result = EEXIST; + break; + } + } + else + { + *filename = '\0'; + *filename_len = 0; + } + + pFileContext->extra_info.upload.file_type |= _FILE_TYPE_LINK; + pClientInfo->file_context.extra_info.upload.trunk_info.path. \ + store_path_index = store_path_index; + if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK) + { + pFileContext->calc_crc32 = false; + pFileContext->calc_file_hash = false; + pFileContext->extra_info.upload.if_gen_filename = true; + pFileContext->extra_info.upload.start_time = g_current_time; + pFileContext->crc32 = rand(); + strcpy(pFileContext->extra_info.upload.file_ext_name, file_ext_name); + storage_format_ext_name(file_ext_name, \ + pFileContext->extra_info.upload.formatted_ext_name); + return storage_trunk_create_link(pTask, src_filename, \ + pSourceFileInfo, bNeedReponse); + } + + pClientInfo->file_context.extra_info.upload.if_sub_path_alloced = false; + result = storage_service_do_create_link(pTask, pSourceFileInfo, \ + stat_buf.st_size, master_filename, \ + prefix_name, file_ext_name, filename, filename_len); + if (result != 0) + { + break; + } + + snprintf(binlog_buff, sizeof(binlog_buff), "%s %s", \ + filename, src_filename); + result = storage_binlog_write(g_current_time, \ + STORAGE_OP_TYPE_SOURCE_CREATE_LINK, binlog_buff); + } while (0); + + if (result == 0) + { + CHECK_AND_WRITE_TO_STAT_FILE3( \ + g_storage_stat.total_create_link_count, \ + g_storage_stat.success_create_link_count, \ + g_storage_stat.last_source_update) + } + else + { + pthread_mutex_lock(&stat_count_thread_lock); + g_storage_stat.total_create_link_count++; + pthread_mutex_unlock(&stat_count_thread_lock); + } + + return result; +} + +/** +pkg format: +Header +8 bytes: master filename len +8 bytes: source filename len +8 bytes: source file signature len +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +FDFS_FILE_PREFIX_MAX_LEN bytes : filename prefix, can be empty +FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.) +master filename len: master filename +source filename len: source filename +source file signature len: source file signature +**/ +static int storage_do_create_link(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + TrackerHeader *pHeader; + char *p; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char prefix_name[FDFS_FILE_PREFIX_MAX_LEN + 1]; + char file_ext_name[FDFS_FILE_EXT_NAME_MAX_LEN + 1]; + char src_filename[128]; + char master_filename[128]; + char filename[128]; + int src_filename_len; + int master_filename_len; + int filename_len; + int len; + int result; + int store_path_index; + SourceFileInfo sourceFileInfo; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + pClientInfo->total_length = sizeof(TrackerHeader); + + do + { + if (nInPackLen <= 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, pTask->client_ip, \ + nInPackLen, 4 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN); + result = EINVAL; + break; + } + + p = pTask->data + sizeof(TrackerHeader); + master_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + src_filename_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + sourceFileInfo.src_file_sig_len = buff2long(p); + p += FDFS_PROTO_PKG_LEN_SIZE; + if (master_filename_len < 0 || master_filename_len >= \ + sizeof(master_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid master filename length: %d", \ + __LINE__, pTask->client_ip, master_filename_len); + result = EINVAL; + break; + } + + if (src_filename_len <= 0 || src_filename_len >= \ + sizeof(src_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid filename length: %d", \ + __LINE__, pTask->client_ip, src_filename_len); + result = EINVAL; + break; + } + + if (sourceFileInfo.src_file_sig_len <= 0 || \ + sourceFileInfo.src_file_sig_len >= \ + sizeof(sourceFileInfo.src_file_sig)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid file signature length: %d", \ + __LINE__, pTask->client_ip, \ + sourceFileInfo.src_file_sig_len); + result = EINVAL; + break; + } + + if (sourceFileInfo.src_file_sig_len != nInPackLen - \ + (3 * FDFS_PROTO_PKG_LEN_SIZE+FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_FILE_PREFIX_MAX_LEN + FDFS_FILE_EXT_NAME_MAX_LEN +\ + master_filename_len + src_filename_len)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid src_file_sig_len : %d", \ + __LINE__, pTask->client_ip, \ + sourceFileInfo.src_file_sig_len); + result = EINVAL; + break; + } + + memcpy(group_name, p, FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + p += FDFS_GROUP_NAME_MAX_LEN; + if (strcmp(group_name, g_group_name) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, group_name: %s " \ + "not correct, should be: %s", \ + __LINE__, pTask->client_ip, \ + group_name, g_group_name); + result = EINVAL; + break; + } + + memcpy(prefix_name, p, FDFS_FILE_PREFIX_MAX_LEN); + *(prefix_name + FDFS_FILE_PREFIX_MAX_LEN) = '\0'; + p += FDFS_FILE_PREFIX_MAX_LEN; + + memcpy(file_ext_name, p, FDFS_FILE_EXT_NAME_MAX_LEN); + *(file_ext_name + FDFS_FILE_EXT_NAME_MAX_LEN) = '\0'; + p += FDFS_FILE_EXT_NAME_MAX_LEN; + + len = master_filename_len + src_filename_len + \ + sourceFileInfo.src_file_sig_len; + if (len > 256) + { + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, invalid pkg length, " \ + "file relative length: %d > %d", \ + __LINE__, pTask->client_ip, len, 256); + result = EINVAL; + break; + } + + if (master_filename_len > 0) + { + memcpy(master_filename, p, master_filename_len); + *(master_filename + master_filename_len) = '\0'; + p += master_filename_len; + } + else + { + *master_filename = '\0'; + } + + memcpy(src_filename, p, src_filename_len); + *(src_filename + src_filename_len) = '\0'; + p += src_filename_len; + + memcpy(sourceFileInfo.src_file_sig, p, sourceFileInfo.src_file_sig_len); + *(sourceFileInfo.src_file_sig + sourceFileInfo.src_file_sig_len) = '\0'; + + if ((result=storage_split_filename_ex(src_filename, \ + &src_filename_len, sourceFileInfo.src_true_filename, \ + &store_path_index)) != 0) + { + break; + } + if ((result=fdfs_check_data_filename( \ + sourceFileInfo.src_true_filename, src_filename_len)) != 0) + { + break; + } + + pClientInfo->file_context.extra_info.upload.trunk_info.path. \ + store_path_index = store_path_index; + result = storage_create_link_core(pTask, \ + &sourceFileInfo, src_filename, \ + master_filename, master_filename_len, \ + prefix_name, file_ext_name, \ + filename, &filename_len, true); + if (result == STORAGE_STATUE_DEAL_FILE) + { + return 0; + } + } while (0); + + if (result == 0) + { + pClientInfo->total_length += FDFS_GROUP_NAME_MAX_LEN + filename_len; + p = pTask->data + sizeof(TrackerHeader); + memcpy(p, g_group_name, FDFS_GROUP_NAME_MAX_LEN); + memcpy(p + FDFS_GROUP_NAME_MAX_LEN, filename, filename_len); + } + + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + + storage_nio_notify(pTask); + + return result; +} + +static int storage_create_link(struct fast_task_info *pTask) +{ + StorageClientInfo *pClientInfo; + StorageFileContext *pFileContext; + char *p; + char src_filename[128]; + char src_true_filename[128]; + int src_filename_len; + int result; + int store_path_index; + int64_t nInPackLen; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pFileContext = &(pClientInfo->file_context); + + nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader); + + if (nInPackLen <= 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + INT64_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + STORAGE_PROTO_CMD_UPLOAD_FILE, pTask->client_ip, \ + nInPackLen, 4 * FDFS_PROTO_PKG_LEN_SIZE + \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_FILE_PREFIX_MAX_LEN + \ + FDFS_FILE_EXT_NAME_MAX_LEN); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader) + FDFS_PROTO_PKG_LEN_SIZE; + src_filename_len = buff2long(p); + if (src_filename_len <= 0 || src_filename_len >= \ + sizeof(src_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length is not correct, " \ + "invalid filename length: %d", \ + __LINE__, pTask->client_ip, src_filename_len); + pClientInfo->total_length = sizeof(TrackerHeader); + return EINVAL; + } + + p += 2 * FDFS_PROTO_PKG_LEN_SIZE + FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_FILE_PREFIX_MAX_LEN + FDFS_FILE_EXT_NAME_MAX_LEN; + memcpy(src_filename, p, src_filename_len); + *(src_filename + src_filename_len) = '\0'; + + if ((result=storage_split_filename_ex(src_filename, \ + &src_filename_len, src_true_filename, &store_path_index)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + pClientInfo->deal_func = storage_do_create_link; + + pFileContext->fd = -1; + pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE; + pFileContext->dio_thread_index = storage_dio_get_thread_index( \ + pTask, store_path_index, pFileContext->op); + + if ((result=storage_dio_queue_push(pTask)) != 0) + { + pClientInfo->total_length = sizeof(TrackerHeader); + return result; + } + + return STORAGE_STATUE_DEAL_FILE; +} + +int fdfs_stat_file_sync_func(void *args) +{ + int result; + + if (last_stat_change_count != g_stat_change_count) + { + if ((result=storage_write_to_stat_file()) == 0) + { + last_stat_change_count = g_stat_change_count; + } + + return result; + } + else + { + return 0; + } +} + +#define ACCESS_LOG_INIT_FIELDS() \ + do \ + { \ + if (g_use_access_log) \ + { \ + *(pClientInfo->file_context.fname2log) = '-'; \ + *(pClientInfo->file_context.fname2log+1)='\0';\ + pClientInfo->request_length = \ + pClientInfo->total_length; \ + gettimeofday(&(pClientInfo->file_context. \ + tv_deal_start), NULL); \ + } \ + } while (0) + +int storage_deal_task(struct fast_task_info *pTask) +{ + TrackerHeader *pHeader; + StorageClientInfo *pClientInfo; + int result; + + pClientInfo = (StorageClientInfo *)pTask->arg; + pHeader = (TrackerHeader *)pTask->data; + + switch(pHeader->cmd) + { + case STORAGE_PROTO_CMD_DOWNLOAD_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_server_download_file(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_DOWNLOAD_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_GET_METADATA: + ACCESS_LOG_INIT_FIELDS(); + result = storage_server_get_metadata(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_GET_METADATA, \ + result); + break; + case STORAGE_PROTO_CMD_UPLOAD_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_upload_file(pTask, false); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_UPLOAD_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_upload_file(pTask, true); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_UPLOAD_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_APPEND_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_append_file(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_APPEND_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_MODIFY_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_modify_file(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_MODIFY_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_TRUNCATE_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_do_truncate_file(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_TRUNCATE_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_upload_slave_file(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_UPLOAD_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_DELETE_FILE: + ACCESS_LOG_INIT_FIELDS(); + result = storage_server_delete_file(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_DELETE_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_SET_METADATA: + ACCESS_LOG_INIT_FIELDS(); + result = storage_server_set_metadata(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_SET_METADATA, \ + result); + break; + case STORAGE_PROTO_CMD_QUERY_FILE_INFO: + ACCESS_LOG_INIT_FIELDS(); + result = storage_server_query_file_info(pTask); + STORAGE_ACCESS_LOG(pTask, \ + ACCESS_LOG_ACTION_QUERY_FILE, \ + result); + break; + case STORAGE_PROTO_CMD_CREATE_LINK: + result = storage_create_link(pTask); + break; + case STORAGE_PROTO_CMD_SYNC_CREATE_FILE: + result = storage_sync_copy_file(pTask, pHeader->cmd); + break; + case STORAGE_PROTO_CMD_SYNC_DELETE_FILE: + result = storage_sync_delete_file(pTask); + break; + case STORAGE_PROTO_CMD_SYNC_UPDATE_FILE: + result = storage_sync_copy_file(pTask, pHeader->cmd); + break; + case STORAGE_PROTO_CMD_SYNC_APPEND_FILE: + result = storage_sync_append_file(pTask); + break; + case STORAGE_PROTO_CMD_SYNC_MODIFY_FILE: + result = storage_sync_modify_file(pTask); + break; + case STORAGE_PROTO_CMD_SYNC_TRUNCATE_FILE: + result = storage_sync_truncate_file(pTask); + break; + case STORAGE_PROTO_CMD_SYNC_CREATE_LINK: + result = storage_sync_link_file(pTask); + break; + case STORAGE_PROTO_CMD_FETCH_ONE_PATH_BINLOG: + result = storage_server_fetch_one_path_binlog(pTask); + break; + case FDFS_PROTO_CMD_QUIT: + add_to_deleted_list(pTask); + return 0; + case FDFS_PROTO_CMD_ACTIVE_TEST: + result = storage_deal_active_test(pTask); + break; + case STORAGE_PROTO_CMD_REPORT_SERVER_ID: + result = storage_server_report_server_id(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_ALLOC_SPACE: + result = storage_server_trunk_alloc_space(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_ALLOC_CONFIRM: + result = storage_server_trunk_alloc_confirm(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_FREE_SPACE: + result = storage_server_trunk_free_space(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG: + result = storage_server_trunk_sync_binlog(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_GET_BINLOG_SIZE: + result = storage_server_trunk_get_binlog_size(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_DELETE_BINLOG_MARKS: + result = storage_server_trunk_delete_binlog_marks(pTask); + break; + case STORAGE_PROTO_CMD_TRUNK_TRUNCATE_BINLOG_FILE: + result = storage_server_trunk_truncate_binlog_file(pTask); + break; + default: + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, unkown cmd: %d", \ + __LINE__, pTask->client_ip, \ + pHeader->cmd); + pClientInfo->total_length = sizeof(TrackerHeader); + result = EINVAL; + break; + } + + if (result != STORAGE_STATUE_DEAL_FILE) + { + pClientInfo->total_offset = 0; + pTask->length = pClientInfo->total_length; + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = STORAGE_PROTO_CMD_RESP; + long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \ + pHeader->pkg_len); + storage_send_add_event(pTask); + } + + return result; +} + diff --git a/storage/storage_service.h b/storage/storage_service.h new file mode 100644 index 0000000..a820184 --- /dev/null +++ b/storage/storage_service.h @@ -0,0 +1,52 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_service.h + +#ifndef _STORAGE_SERVICE_H_ +#define _STORAGE_SERVICE_H_ + +#define STORAGE_CREATE_FLAG_NONE 0 +#define STORAGE_CREATE_FLAG_FILE 1 +#define STORAGE_CREATE_FLAG_LINK 2 + +#define STORAGE_DELETE_FLAG_NONE 0 +#define STORAGE_DELETE_FLAG_FILE 1 +#define STORAGE_DELETE_FLAG_LINK 2 + +#include "logger.h" +#include "fdfs_define.h" +#include "fast_task_queue.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_storage_thread_count; +extern pthread_mutex_t g_storage_thread_lock; + +int storage_service_init(); +void storage_service_destroy(); + +int fdfs_stat_file_sync_func(void *args); +int storage_deal_task(struct fast_task_info *pTask); + +void storage_nio_notify(struct fast_task_info *pTask); +void storage_accept_loop(int server_sock); +int storage_terminate_threads(); + +int storage_get_storage_path_index(int *store_path_index); + +void storage_get_store_path(const char *filename, const int filename_len, \ + int *sub_path_high, int *sub_path_low); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/storage_sync.c b/storage/storage_sync.c new file mode 100644 index 0000000..e8ce0c5 --- /dev/null +++ b/storage/storage_sync.c @@ -0,0 +1,3089 @@ +/** * Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_sync.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "ini_file_reader.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_ip_changed_dealer.h" +#include "tracker_client_thread.h" +#include "storage_client.h" +#include "storage_sync.h" +#include "trunk_mem.h" + +#define SYNC_BINLOG_FILE_MAX_SIZE 1024 * 1024 * 1024 +#define SYNC_BINLOG_FILE_PREFIX "binlog" +#define SYNC_BINLOG_INDEX_FILENAME SYNC_BINLOG_FILE_PREFIX".index" +#define SYNC_MARK_FILE_EXT ".mark" +#define SYNC_BINLOG_FILE_EXT_FMT ".%03d" +#define SYNC_DIR_NAME "sync" +#define MARK_ITEM_BINLOG_FILE_INDEX "binlog_index" +#define MARK_ITEM_BINLOG_FILE_OFFSET "binlog_offset" +#define MARK_ITEM_NEED_SYNC_OLD "need_sync_old" +#define MARK_ITEM_SYNC_OLD_DONE "sync_old_done" +#define MARK_ITEM_UNTIL_TIMESTAMP "until_timestamp" +#define MARK_ITEM_SCAN_ROW_COUNT "scan_row_count" +#define MARK_ITEM_SYNC_ROW_COUNT "sync_row_count" +#define SYNC_BINLOG_WRITE_BUFF_SIZE (16 * 1024) + +int g_binlog_fd = -1; +int g_binlog_index = 0; +static int64_t binlog_file_size = 0; + +int g_storage_sync_thread_count = 0; +static pthread_mutex_t sync_thread_lock; +static char *binlog_write_cache_buff = NULL; +static int binlog_write_cache_len = 0; +static int binlog_write_version = 1; + +/* save sync thread ids */ +static pthread_t *sync_tids = NULL; + +static int storage_write_to_mark_file(StorageBinLogReader *pReader); +static int storage_binlog_reader_skip(StorageBinLogReader *pReader); +static int storage_binlog_fsync(const bool bNeedLock); +static int storage_binlog_preread(StorageBinLogReader *pReader); + +/** +8 bytes: filename bytes +8 bytes: file size +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +file size bytes: file content +**/ +static int storage_sync_copy_file(ConnectionInfo *pStorageServer, \ + StorageBinLogReader *pReader, const StorageBinLogRecord *pRecord, \ + char proto_cmd) +{ + TrackerHeader *pHeader; + char *p; + char *pBuff; + char full_filename[MAX_PATH_SIZE]; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+256]; + char in_buff[1]; + struct stat stat_buf; + FDFSTrunkFullInfo trunkInfo; + FDFSTrunkHeader trunkHeader; + int64_t file_offset; + int64_t in_bytes; + int64_t total_send_bytes; + int result; + bool need_sync_file; + + if ((result=trunk_file_stat(pRecord->store_path_index, \ + pRecord->true_filename, pRecord->true_filename_len, \ + &stat_buf, &trunkInfo, &trunkHeader)) != 0) + { + if (result == ENOENT) + { + if(pRecord->op_type==STORAGE_OP_TYPE_SOURCE_CREATE_FILE) + { + logDebug("file: "__FILE__", line: %d, " \ + "sync data file, logic file: %s " \ + "not exists, maybe deleted later?", \ + __LINE__, pRecord->filename); + } + + return 0; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "call stat fail, logic file: %s, "\ + "error no: %d, error info: %s", \ + __LINE__, pRecord->filename, \ + result, STRERROR(result)); + return result; + } + } + + need_sync_file = true; + if (pReader->last_file_exist && proto_cmd == \ + STORAGE_PROTO_CMD_SYNC_CREATE_FILE) + { + FDFSFileInfo file_info; + result = storage_query_file_info_ex(NULL, \ + pStorageServer, g_group_name, \ + pRecord->filename, &file_info, true); + if (result == 0) + { + if (file_info.file_size == stat_buf.st_size) + { + logDebug("file: "__FILE__", line: %d, " \ + "sync data file, logic file: %s " \ + "on dest server %s:%d already exists, "\ + "and same as mine, ignore it", \ + __LINE__, pRecord->filename, \ + pStorageServer->ip_addr, \ + pStorageServer->port); + need_sync_file = false; + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "sync data file, logic file: %s " \ + "on dest server %s:%d already exists, "\ + "but file size: "INT64_PRINTF_FORMAT \ + " not same as mine: "OFF_PRINTF_FORMAT\ + ", need re-sync it", __LINE__, \ + pRecord->filename, pStorageServer->ip_addr,\ + pStorageServer->port, \ + file_info.file_size, stat_buf.st_size); + + proto_cmd = STORAGE_PROTO_CMD_SYNC_UPDATE_FILE; + } + } + else if (result != ENOENT) + { + return result; + } + } + + if (IS_TRUNK_FILE_BY_ID(trunkInfo)) + { + file_offset = TRUNK_FILE_START_OFFSET(trunkInfo); + trunk_get_full_filename((&trunkInfo), full_filename, \ + sizeof(full_filename)); + } + else + { + file_offset = 0; + sprintf(full_filename, "%s/data/%s", \ + g_fdfs_store_paths.paths[pRecord->store_path_index], \ + pRecord->true_filename); + } + + total_send_bytes = 0; + //printf("sync create file: %s\n", pRecord->filename); + do + { + int64_t body_len; + + pHeader = (TrackerHeader *)out_buff; + memset(pHeader, 0, sizeof(TrackerHeader)); + + body_len = 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + \ + pRecord->filename_len; + if (need_sync_file) + { + body_len += stat_buf.st_size; + } + + long2buff(body_len, pHeader->pkg_len); + pHeader->cmd = proto_cmd; + pHeader->status = need_sync_file ? 0 : EEXIST; + + p = out_buff + sizeof(TrackerHeader); + + long2buff(pRecord->filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(stat_buf.st_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + int2buff(pRecord->timestamp, p); + p += 4; + + sprintf(p, "%s", g_group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + memcpy(p, pRecord->filename, pRecord->filename_len); + p += pRecord->filename_len; + + if((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + if (need_sync_file && (stat_buf.st_size > 0) && \ + ((result=tcpsendfile_ex(pStorageServer->sock, \ + full_filename, file_offset, stat_buf.st_size, \ + g_fdfs_network_timeout, &total_send_bytes)) != 0)) + { + logError("file: "__FILE__", line: %d, " \ + "sync data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + pBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pBuff, 0, &in_bytes)) != 0) + { + break; + } + } while (0); + + pthread_mutex_lock(&sync_thread_lock); + g_storage_stat.total_sync_out_bytes += total_send_bytes; + if (result == 0) + { + g_storage_stat.success_sync_out_bytes += total_send_bytes; + } + pthread_mutex_unlock(&sync_thread_lock); + + if (result == EEXIST) + { + if (need_sync_file && pRecord->op_type == \ + STORAGE_OP_TYPE_SOURCE_CREATE_FILE) + { + logWarning("file: "__FILE__", line: %d, " \ + "storage server ip: %s:%d, data file: %s " \ + "already exists, maybe some mistake?", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, pRecord->filename); + } + + pReader->last_file_exist = true; + return 0; + } + else if (result == 0) + { + pReader->last_file_exist = false; + return 0; + } + else + { + return result; + } +} + +/** +8 bytes: filename bytes +8 bytes: start offset +8 bytes: append length +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +file size bytes: file content +**/ +static int storage_sync_modify_file(ConnectionInfo *pStorageServer, \ + StorageBinLogReader *pReader, StorageBinLogRecord *pRecord, \ + const char cmd) +{ +#define FIELD_COUNT 3 + TrackerHeader *pHeader; + char *p; + char *pBuff; + char *fields[FIELD_COUNT]; + char full_filename[MAX_PATH_SIZE]; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+256]; + char in_buff[1]; + struct stat stat_buf; + int64_t in_bytes; + int64_t total_send_bytes; + int64_t start_offset; + int64_t modify_length; + int result; + int count; + + if ((count=splitEx(pRecord->filename, ' ', fields, FIELD_COUNT)) \ + != FIELD_COUNT) + { + logError("file: "__FILE__", line: %d, " \ + "the format of binlog not correct, filename: %s", \ + __LINE__, pRecord->filename); + return EINVAL; + } + + start_offset = strtoll((fields[1]), NULL, 10); + modify_length = strtoll((fields[2]), NULL, 10); + + pRecord->filename_len = strlen(pRecord->filename); + pRecord->true_filename_len = pRecord->filename_len; + if ((result=storage_split_filename_ex(pRecord->filename, \ + &pRecord->true_filename_len, pRecord->true_filename, \ + &pRecord->store_path_index)) != 0) + { + return result; + } + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[pRecord->store_path_index], \ + pRecord->true_filename); + if (lstat(full_filename, &stat_buf) != 0) + { + if (errno == ENOENT) + { + logDebug("file: "__FILE__", line: %d, " \ + "sync appender file, file: %s not exists, "\ + "maybe deleted later?", \ + __LINE__, full_filename); + + return 0; + } + else + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "call stat fail, appender file: %s, "\ + "error no: %d, error info: %s", \ + __LINE__, full_filename, \ + result, STRERROR(result)); + return result; + } + } + + if (stat_buf.st_size < start_offset + modify_length) + { + logWarning("file: "__FILE__", line: %d, " \ + "appender file: %s 'size: "INT64_PRINTF_FORMAT \ + " < "INT64_PRINTF_FORMAT", maybe some mistakes " \ + "happened, skip sync this appender file", __LINE__, \ + full_filename, stat_buf.st_size, \ + start_offset + modify_length); + + return 0; + } + + total_send_bytes = 0; + //printf("sync create file: %s\n", pRecord->filename); + do + { + int64_t body_len; + + pHeader = (TrackerHeader *)out_buff; + memset(pHeader, 0, sizeof(TrackerHeader)); + + body_len = 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + \ + pRecord->filename_len + modify_length; + + long2buff(body_len, pHeader->pkg_len); + pHeader->cmd = cmd; + pHeader->status = 0; + + p = out_buff + sizeof(TrackerHeader); + + long2buff(pRecord->filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(start_offset, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(modify_length, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + int2buff(pRecord->timestamp, p); + p += 4; + + sprintf(p, "%s", g_group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + memcpy(p, pRecord->filename, pRecord->filename_len); + p += pRecord->filename_len; + + if((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + if ((result=tcpsendfile_ex(pStorageServer->sock, \ + full_filename, start_offset, modify_length, \ + g_fdfs_network_timeout, &total_send_bytes)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + pBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pBuff, 0, &in_bytes)) != 0) + { + break; + } + } while (0); + + pthread_mutex_lock(&sync_thread_lock); + g_storage_stat.total_sync_out_bytes += total_send_bytes; + if (result == 0) + { + g_storage_stat.success_sync_out_bytes += total_send_bytes; + } + pthread_mutex_unlock(&sync_thread_lock); + + return result == EEXIST ? 0 : result; +} + +/** +8 bytes: filename bytes +8 bytes: old file size +8 bytes: new file size +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +filename bytes : filename +**/ +static int storage_sync_truncate_file(ConnectionInfo *pStorageServer, \ + StorageBinLogReader *pReader, StorageBinLogRecord *pRecord) +{ +#define FIELD_COUNT 3 + TrackerHeader *pHeader; + char *p; + char *pBuff; + char *fields[FIELD_COUNT]; + char full_filename[MAX_PATH_SIZE]; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+256]; + char in_buff[1]; + struct stat stat_buf; + int64_t in_bytes; + int64_t old_file_size; + int64_t new_file_size; + int result; + int count; + + if ((count=splitEx(pRecord->filename, ' ', fields, FIELD_COUNT)) \ + != FIELD_COUNT) + { + logError("file: "__FILE__", line: %d, " \ + "the format of binlog not correct, filename: %s", \ + __LINE__, pRecord->filename); + return EINVAL; + } + + old_file_size = strtoll((fields[1]), NULL, 10); + new_file_size = strtoll((fields[2]), NULL, 10); + + pRecord->filename_len = strlen(pRecord->filename); + pRecord->true_filename_len = pRecord->filename_len; + if ((result=storage_split_filename_ex(pRecord->filename, \ + &pRecord->true_filename_len, pRecord->true_filename, \ + &pRecord->store_path_index)) != 0) + { + return result; + } + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[pRecord->store_path_index], \ + pRecord->true_filename); + if (lstat(full_filename, &stat_buf) != 0) + { + if (errno == ENOENT) + { + logDebug("file: "__FILE__", line: %d, " \ + "sync appender file, file: %s not exists, "\ + "maybe deleted later?", \ + __LINE__, full_filename); + + return 0; + } + else + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "call stat fail, appender file: %s, "\ + "error no: %d, error info: %s", \ + __LINE__, full_filename, \ + result, STRERROR(result)); + return result; + } + } + + if (stat_buf.st_size != new_file_size) + { + logDebug("file: "__FILE__", line: %d, " \ + "appender file: %s 'size: "INT64_PRINTF_FORMAT \ + " != "INT64_PRINTF_FORMAT", maybe append/modify later",\ + __LINE__, full_filename, stat_buf.st_size, + new_file_size); + } + + do + { + int64_t body_len; + + pHeader = (TrackerHeader *)out_buff; + memset(pHeader, 0, sizeof(TrackerHeader)); + + body_len = 3 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + \ + pRecord->filename_len; + + long2buff(body_len, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_SYNC_TRUNCATE_FILE; + pHeader->status = 0; + + p = out_buff + sizeof(TrackerHeader); + + long2buff(pRecord->filename_len, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(old_file_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(new_file_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + int2buff(pRecord->timestamp, p); + p += 4; + + sprintf(p, "%s", g_group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + memcpy(p, pRecord->filename, pRecord->filename_len); + p += pRecord->filename_len; + + if((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + p - out_buff, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + + break; + } + + pBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pBuff, 0, &in_bytes)) != 0) + { + break; + } + } while (0); + + return result == EEXIST ? 0 : result; +} + +/** +send pkg format: +4 bytes: source delete timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +remain bytes: filename +**/ +static int storage_sync_delete_file(ConnectionInfo *pStorageServer, \ + const StorageBinLogRecord *pRecord) +{ + TrackerHeader *pHeader; + char out_buff[sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN+256]; + struct stat stat_buf; + FDFSTrunkFullInfo trunkInfo; + FDFSTrunkHeader trunkHeader; + char in_buff[1]; + char *pBuff; + int64_t in_bytes; + int result; + + if ((result=trunk_file_stat(pRecord->store_path_index, \ + pRecord->true_filename, pRecord->true_filename_len, \ + &stat_buf, &trunkInfo, &trunkHeader)) == 0) + { + if (pRecord->op_type == STORAGE_OP_TYPE_SOURCE_DELETE_FILE) + { + logWarning("file: "__FILE__", line: %d, " \ + "sync data file, logic file: %s exists, " \ + "maybe created later?", \ + __LINE__, pRecord->filename); + } + + return 0; + } + + memset(out_buff, 0, sizeof(out_buff)); + int2buff(pRecord->timestamp, out_buff + sizeof(TrackerHeader)); + memcpy(out_buff + sizeof(TrackerHeader) + 4, g_group_name, \ + sizeof(g_group_name)); + memcpy(out_buff + sizeof(TrackerHeader) + 4 + FDFS_GROUP_NAME_MAX_LEN, \ + pRecord->filename, pRecord->filename_len); + + pHeader = (TrackerHeader *)out_buff; + long2buff(4 + FDFS_GROUP_NAME_MAX_LEN + pRecord->filename_len, \ + pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_SYNC_DELETE_FILE; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(TrackerHeader) + 4 + FDFS_GROUP_NAME_MAX_LEN + \ + pRecord->filename_len, g_fdfs_network_timeout)) != 0) + { + logError("FILE: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = in_buff; + result = fdfs_recv_response(pStorageServer, &pBuff, 0, &in_bytes); + if (result == ENOENT) + { + result = 0; + } + + return result; +} + +/** +FDFS_STORAGE_ID_MAX_SIZE bytes: my server id +**/ +static int storage_report_my_server_id(ConnectionInfo *pStorageServer) +{ + int result; + TrackerHeader *pHeader; + char out_buff[sizeof(TrackerHeader) + FDFS_STORAGE_ID_MAX_SIZE]; + char in_buff[1]; + char *pBuff; + int64_t in_bytes; + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + + long2buff(IP_ADDRESS_SIZE, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_REPORT_SERVER_ID; + strcpy(out_buff + sizeof(TrackerHeader), g_my_server_id_str); + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_STORAGE_ID_MAX_SIZE, \ + g_fdfs_network_timeout)) != 0) + { + logError("FILE: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = in_buff; + return fdfs_recv_response(pStorageServer, &pBuff, 0, &in_bytes); +} + +/** +8 bytes: dest(link) filename length +8 bytes: source filename length +4 bytes: source op timestamp +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +dest filename length: dest filename +source filename length: source filename +**/ +static int storage_sync_link_file(ConnectionInfo *pStorageServer, \ + StorageBinLogRecord *pRecord) +{ + TrackerHeader *pHeader; + int result; + char out_buff[sizeof(TrackerHeader) + 2 * FDFS_PROTO_PKG_LEN_SIZE + \ + 4 + FDFS_GROUP_NAME_MAX_LEN + 256]; + char in_buff[1]; + FDFSTrunkFullInfo trunkInfo; + FDFSTrunkHeader trunkHeader; + int out_body_len; + int64_t in_bytes; + char *pBuff; + struct stat stat_buf; + int fd; + + fd = -1; + if ((result=trunk_file_lstat_ex(pRecord->store_path_index, \ + pRecord->true_filename, pRecord->true_filename_len, \ + &stat_buf, &trunkInfo, &trunkHeader, &fd)) != 0) + { + if (result == ENOENT) + { + if (pRecord->op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK) + { + logDebug("file: "__FILE__", line: %d, " \ + "sync data file, logic file: %s does not " \ + "exist, maybe delete later?", \ + __LINE__, pRecord->filename); + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "call stat fail, logic file: %s, "\ + "error no: %d, error info: %s", \ + __LINE__, pRecord->filename, \ + result, STRERROR(result)); + } + + return 0; + } + + if (!S_ISLNK(stat_buf.st_mode)) + { + if (fd >= 0) + { + close(fd); + } + + if (pRecord->op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK) + { + logWarning("file: "__FILE__", line: %d, " \ + "sync data file, logic file %s is not " \ + "a symbol link, maybe create later?", \ + __LINE__, pRecord->filename); + } + + return 0; + } + + if (pRecord->src_filename_len > 0) + { + if (fd >= 0) + { + close(fd); + } + } + else if (IS_TRUNK_FILE_BY_ID(trunkInfo)) + { + result = trunk_file_get_content(&trunkInfo, \ + stat_buf.st_size, &fd, pRecord->src_filename, \ + sizeof(pRecord->src_filename)); + close(fd); + + if (result != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "logic file: %s, get file content fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pRecord->filename, \ + result, STRERROR(result)); + return 0; + } + + pRecord->src_filename_len = stat_buf.st_size; + *(pRecord->src_filename + pRecord->src_filename_len) = '\0'; + } + else + { + char full_filename[MAX_PATH_SIZE]; + char src_full_filename[MAX_PATH_SIZE]; + char *p; + char *pSrcFilename; + int src_path_index; + int src_filename_len; + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/%s", g_fdfs_store_paths.paths[pRecord->store_path_index], \ + pRecord->true_filename); + src_filename_len = readlink(full_filename, src_full_filename, \ + sizeof(src_full_filename) - 1); + if (src_filename_len <= 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "data file: %s, readlink fail, "\ + "errno: %d, error info: %s", \ + __LINE__, src_full_filename, errno, STRERROR(errno)); + return 0; + } + *(src_full_filename + src_filename_len) = '\0'; + + pSrcFilename = strstr(src_full_filename, "/data/"); + if (pSrcFilename == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "source data file: %s is invalid", \ + __LINE__, src_full_filename); + return EINVAL; + } + + pSrcFilename += 6; + p = strstr(pSrcFilename, "/data/"); + while (p != NULL) + { + pSrcFilename = p + 6; + p = strstr(pSrcFilename, "/data/"); + } + + if (g_fdfs_store_paths.count == 1) + { + src_path_index = 0; + } + else + { + *(pSrcFilename - 6) = '\0'; + + for (src_path_index=0; src_path_indexsrc_filename_len = src_filename_len - (pSrcFilename - \ + src_full_filename) + 4; + if (pRecord->src_filename_len >= sizeof(pRecord->src_filename)) + { + logError("file: "__FILE__", line: %d, " \ + "source data file: %s is invalid", \ + __LINE__, src_full_filename); + return EINVAL; + } + + sprintf(pRecord->src_filename, "%c"FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + FDFS_STORAGE_STORE_PATH_PREFIX_CHAR, \ + src_path_index, pSrcFilename); + } + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + long2buff(pRecord->filename_len, out_buff + sizeof(TrackerHeader)); + long2buff(pRecord->src_filename_len, out_buff + sizeof(TrackerHeader) + \ + FDFS_PROTO_PKG_LEN_SIZE); + int2buff(pRecord->timestamp, out_buff + sizeof(TrackerHeader) + \ + 2 * FDFS_PROTO_PKG_LEN_SIZE); + sprintf(out_buff + sizeof(TrackerHeader) + 2 * FDFS_PROTO_PKG_LEN_SIZE\ + + 4, "%s", g_group_name); + memcpy(out_buff + sizeof(TrackerHeader) + 2 * FDFS_PROTO_PKG_LEN_SIZE \ + + 4 + FDFS_GROUP_NAME_MAX_LEN, \ + pRecord->filename, pRecord->filename_len); + memcpy(out_buff + sizeof(TrackerHeader) + 2 * FDFS_PROTO_PKG_LEN_SIZE \ + + 4 + FDFS_GROUP_NAME_MAX_LEN + pRecord->filename_len, \ + pRecord->src_filename, pRecord->src_filename_len); + + out_body_len = 2 * FDFS_PROTO_PKG_LEN_SIZE + 4 + \ + FDFS_GROUP_NAME_MAX_LEN + pRecord->filename_len + \ + pRecord->src_filename_len; + long2buff(out_body_len, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_SYNC_CREATE_LINK; + + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(TrackerHeader) + out_body_len, \ + g_fdfs_network_timeout)) != 0) + { + logError("FILE: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = in_buff; + result = fdfs_recv_response(pStorageServer, &pBuff, 0, &in_bytes); + if (result == ENOENT) + { + result = 0; + } + + return result; +} + +#define STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord) \ + if ((!pReader->need_sync_old) || pReader->sync_old_done || \ + (pRecord->timestamp > pReader->until_timestamp)) \ + { \ + return 0; \ + } \ + +static int storage_sync_data(StorageBinLogReader *pReader, \ + ConnectionInfo *pStorageServer, \ + StorageBinLogRecord *pRecord) +{ + int result; + switch(pRecord->op_type) + { + case STORAGE_OP_TYPE_SOURCE_CREATE_FILE: + result = storage_sync_copy_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_CREATE_FILE); + break; + case STORAGE_OP_TYPE_SOURCE_DELETE_FILE: + result = storage_sync_delete_file( \ + pStorageServer, pRecord); + break; + case STORAGE_OP_TYPE_SOURCE_UPDATE_FILE: + result = storage_sync_copy_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_UPDATE_FILE); + break; + case STORAGE_OP_TYPE_SOURCE_APPEND_FILE: + result = storage_sync_modify_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_APPEND_FILE); + if (result == ENOENT) //resync appender file + { + result = storage_sync_copy_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_UPDATE_FILE); + } + break; + case STORAGE_OP_TYPE_SOURCE_MODIFY_FILE: + result = storage_sync_modify_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_MODIFY_FILE); + if (result == ENOENT) //resync appender file + { + result = storage_sync_copy_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_UPDATE_FILE); + } + break; + case STORAGE_OP_TYPE_SOURCE_TRUNCATE_FILE: + result = storage_sync_truncate_file(pStorageServer, \ + pReader, pRecord); + break; + case STORAGE_OP_TYPE_SOURCE_CREATE_LINK: + result = storage_sync_link_file(pStorageServer, \ + pRecord); + break; + case STORAGE_OP_TYPE_REPLICA_CREATE_FILE: + STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord) + result = storage_sync_copy_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_CREATE_FILE); + break; + case STORAGE_OP_TYPE_REPLICA_DELETE_FILE: + STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord) + result = storage_sync_delete_file( \ + pStorageServer, pRecord); + break; + case STORAGE_OP_TYPE_REPLICA_UPDATE_FILE: + STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord) + result = storage_sync_copy_file(pStorageServer, \ + pReader, pRecord, \ + STORAGE_PROTO_CMD_SYNC_UPDATE_FILE); + break; + case STORAGE_OP_TYPE_REPLICA_CREATE_LINK: + STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord) + result = storage_sync_link_file(pStorageServer, \ + pRecord); + break; + case STORAGE_OP_TYPE_REPLICA_APPEND_FILE: + return 0; + case STORAGE_OP_TYPE_REPLICA_MODIFY_FILE: + return 0; + case STORAGE_OP_TYPE_REPLICA_TRUNCATE_FILE: + return 0; + default: + logError("file: "__FILE__", line: %d, " \ + "invalid file operation type: %d", \ + __LINE__, pRecord->op_type); + return EINVAL; + } + + if (result == 0) + { + pReader->sync_row_count++; + + if (pReader->sync_row_count - pReader->last_sync_rows >= \ + g_write_mark_file_freq) + { + if ((result=storage_write_to_mark_file(pReader)) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_write_to_mark_file " \ + "fail, program exit!", __LINE__); + g_continue_flag = false; + return result; + } + } + } + + return result; +} + +static int write_to_binlog_index(const int binlog_index) +{ + char full_filename[MAX_PATH_SIZE]; + char buff[16]; + int fd; + int len; + + snprintf(full_filename, sizeof(full_filename), \ + "%s/data/"SYNC_DIR_NAME"/%s", g_fdfs_base_path, \ + SYNC_BINLOG_INDEX_FILENAME); + if ((fd=open(full_filename, O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + len = sprintf(buff, "%d", binlog_index); + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + close(fd); + return errno != 0 ? errno : EIO; + } + + close(fd); + + STORAGE_CHOWN(full_filename, geteuid(), getegid()) + + return 0; +} + +static char *get_writable_binlog_filename(char *full_filename) +{ + static char buff[MAX_PATH_SIZE]; + + if (full_filename == NULL) + { + full_filename = buff; + } + + snprintf(full_filename, MAX_PATH_SIZE, \ + "%s/data/"SYNC_DIR_NAME"/"SYNC_BINLOG_FILE_PREFIX"" \ + SYNC_BINLOG_FILE_EXT_FMT, \ + g_fdfs_base_path, g_binlog_index); + return full_filename; +} + +static char *get_writable_binlog_filename1(char *full_filename, \ + const int binlog_index) +{ + snprintf(full_filename, MAX_PATH_SIZE, \ + "%s/data/"SYNC_DIR_NAME"/"SYNC_BINLOG_FILE_PREFIX"" \ + SYNC_BINLOG_FILE_EXT_FMT, \ + g_fdfs_base_path, binlog_index); + return full_filename; +} + +static int open_next_writable_binlog() +{ + char full_filename[MAX_PATH_SIZE]; + + if (g_binlog_fd >= 0) + { + close(g_binlog_fd); + g_binlog_fd = -1; + } + + get_writable_binlog_filename1(full_filename, g_binlog_index + 1); + if (fileExists(full_filename)) + { + if (unlink(full_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "unlink file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + logError("file: "__FILE__", line: %d, " \ + "binlog file \"%s\" already exists, truncate", \ + __LINE__, full_filename); + } + + g_binlog_fd = open(full_filename, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (g_binlog_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + STORAGE_FCHOWN(g_binlog_fd, full_filename, geteuid(), getegid()) + + g_binlog_index++; + return 0; +} + +int storage_sync_init() +{ + char data_path[MAX_PATH_SIZE]; + char sync_path[MAX_PATH_SIZE]; + char full_filename[MAX_PATH_SIZE]; + char file_buff[64]; + int bytes; + int result; + int fd; + + snprintf(data_path, sizeof(data_path), "%s/data", g_fdfs_base_path); + if (!fileExists(data_path)) + { + if (mkdir(data_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + STORAGE_CHOWN(data_path, geteuid(), getegid()) + } + + snprintf(sync_path, sizeof(sync_path), \ + "%s/"SYNC_DIR_NAME, data_path); + if (!fileExists(sync_path)) + { + if (mkdir(sync_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, sync_path, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + STORAGE_CHOWN(sync_path, geteuid(), getegid()) + } + + binlog_write_cache_buff = (char *)malloc(SYNC_BINLOG_WRITE_BUFF_SIZE); + if (binlog_write_cache_buff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, SYNC_BINLOG_WRITE_BUFF_SIZE, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + snprintf(full_filename, sizeof(full_filename), \ + "%s/%s", sync_path, SYNC_BINLOG_INDEX_FILENAME); + if ((fd=open(full_filename, O_RDONLY)) >= 0) + { + bytes = read(fd, file_buff, sizeof(file_buff) - 1); + close(fd); + if (bytes <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "read file \"%s\" fail, bytes read: %d", \ + __LINE__, full_filename, bytes); + return errno != 0 ? errno : EIO; + } + + file_buff[bytes] = '\0'; + g_binlog_index = atoi(file_buff); + if (g_binlog_index < 0) + { + logError("file: "__FILE__", line: %d, " \ + "in file \"%s\", binlog_index: %d < 0", \ + __LINE__, full_filename, g_binlog_index); + return EINVAL; + } + } + else + { + g_binlog_index = 0; + if ((result=write_to_binlog_index(g_binlog_index)) != 0) + { + return result; + } + } + + get_writable_binlog_filename(full_filename); + g_binlog_fd = open(full_filename, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (g_binlog_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + binlog_file_size = lseek(g_binlog_fd, 0, SEEK_END); + if (binlog_file_size < 0) + { + logError("file: "__FILE__", line: %d, " \ + "ftell file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + storage_sync_destroy(); + return errno != 0 ? errno : EIO; + } + + STORAGE_FCHOWN(g_binlog_fd, full_filename, geteuid(), getegid()) + + /* + //printf("full_filename=%s, binlog_file_size=%d\n", \ + full_filename, binlog_file_size); + */ + + if ((result=init_pthread_lock(&sync_thread_lock)) != 0) + { + return result; + } + + load_local_host_ip_addrs(); + + return 0; +} + +int storage_sync_destroy() +{ + int result; + + if (g_binlog_fd >= 0) + { + storage_binlog_fsync(true); + close(g_binlog_fd); + g_binlog_fd = -1; + } + + if (binlog_write_cache_buff != NULL) + { + free(binlog_write_cache_buff); + binlog_write_cache_buff = NULL; + if ((result=pthread_mutex_destroy(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_destroy fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + return 0; +} + +int kill_storage_sync_threads() +{ + int result; + int kill_res; + + if (sync_tids == NULL) + { + return 0; + } + + if ((result=pthread_mutex_lock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + kill_res = kill_work_threads(sync_tids, g_storage_sync_thread_count); + + if ((result=pthread_mutex_unlock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + while (g_storage_sync_thread_count > 0) + { + usleep(50000); + } + + return kill_res; +} + +int fdfs_binlog_sync_func(void *args) +{ + if (binlog_write_cache_len > 0) + { + return storage_binlog_fsync(true); + } + else + { + return 0; + } +} + +static int storage_binlog_fsync(const bool bNeedLock) +{ + int result; + int write_ret; + + if (bNeedLock && (result=pthread_mutex_lock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if (binlog_write_cache_len == 0) //ignore + { + write_ret = 0; //skip + } + else if (write(g_binlog_fd, binlog_write_cache_buff, \ + binlog_write_cache_len) != binlog_write_cache_len) + { + logError("file: "__FILE__", line: %d, " \ + "write to binlog file \"%s\" fail, fd=%d, " \ + "errno: %d, error info: %s", \ + __LINE__, get_writable_binlog_filename(NULL), \ + g_binlog_fd, errno, STRERROR(errno)); + write_ret = errno != 0 ? errno : EIO; + } + else if (fsync(g_binlog_fd) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync to binlog file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, get_writable_binlog_filename(NULL), \ + errno, STRERROR(errno)); + write_ret = errno != 0 ? errno : EIO; + } + else + { + binlog_file_size += binlog_write_cache_len; + if (binlog_file_size >= SYNC_BINLOG_FILE_MAX_SIZE) + { + if ((write_ret=write_to_binlog_index( \ + g_binlog_index + 1)) == 0) + { + write_ret = open_next_writable_binlog(); + } + + binlog_file_size = 0; + if (write_ret != 0) + { + g_continue_flag = false; + logCrit("file: "__FILE__", line: %d, " \ + "open binlog file \"%s\" fail, " \ + "program exit!", \ + __LINE__, \ + get_writable_binlog_filename(NULL)); + } + } + else + { + write_ret = 0; + } + } + + binlog_write_version++; + binlog_write_cache_len = 0; //reset cache buff + + if (bNeedLock && (result=pthread_mutex_unlock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return write_ret; +} + +int storage_binlog_write_ex(const int timestamp, const char op_type, \ + const char *filename, const char *extra) +{ + int result; + int write_ret; + + if ((result=pthread_mutex_lock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if (extra != NULL) + { + binlog_write_cache_len += sprintf(binlog_write_cache_buff + \ + binlog_write_cache_len, "%d %c %s %s\n",\ + timestamp, op_type, filename, extra); + } + else + { + binlog_write_cache_len += sprintf(binlog_write_cache_buff + \ + binlog_write_cache_len, "%d %c %s\n", \ + timestamp, op_type, filename); + } + + //check if buff full + if (SYNC_BINLOG_WRITE_BUFF_SIZE - binlog_write_cache_len < 256) + { + write_ret = storage_binlog_fsync(false); //sync to disk + } + else + { + write_ret = 0; + } + + if ((result=pthread_mutex_unlock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return write_ret; +} + +static char *get_binlog_readable_filename(const void *pArg, \ + char *full_filename) +{ + const StorageBinLogReader *pReader; + static char buff[MAX_PATH_SIZE]; + + pReader = (const StorageBinLogReader *)pArg; + if (full_filename == NULL) + { + full_filename = buff; + } + + snprintf(full_filename, MAX_PATH_SIZE, \ + "%s/data/"SYNC_DIR_NAME"/"SYNC_BINLOG_FILE_PREFIX"" \ + SYNC_BINLOG_FILE_EXT_FMT, \ + g_fdfs_base_path, pReader->binlog_index); + return full_filename; +} + +int storage_open_readable_binlog(StorageBinLogReader *pReader, \ + get_filename_func filename_func, const void *pArg) +{ + char full_filename[MAX_PATH_SIZE]; + + if (pReader->binlog_fd >= 0) + { + close(pReader->binlog_fd); + } + + filename_func(pArg, full_filename); + pReader->binlog_fd = open(full_filename, O_RDONLY); + if (pReader->binlog_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open binlog file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (pReader->binlog_offset > 0 && \ + lseek(pReader->binlog_fd, pReader->binlog_offset, SEEK_SET) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "seek binlog file \"%s\" fail, file offset=" \ + INT64_PRINTF_FORMAT", errno: %d, error info: %s", \ + __LINE__, full_filename, pReader->binlog_offset, \ + errno, STRERROR(errno)); + + close(pReader->binlog_fd); + pReader->binlog_fd = -1; + return errno != 0 ? errno : ESPIPE; + } + + return 0; +} + +static char *get_mark_filename_by_id_and_port(const char *storage_id, \ + const int port, char *full_filename, const int filename_size) +{ + if (g_use_storage_id) + { + snprintf(full_filename, filename_size, \ + "%s/data/"SYNC_DIR_NAME"/%s%s", g_fdfs_base_path, \ + storage_id, SYNC_MARK_FILE_EXT); + } + else + { + snprintf(full_filename, filename_size, \ + "%s/data/"SYNC_DIR_NAME"/%s_%d%s", g_fdfs_base_path, \ + storage_id, port, SYNC_MARK_FILE_EXT); + } + return full_filename; +} + +static char *get_mark_filename_by_ip_and_port(const char *ip_addr, \ + const int port, char *full_filename, const int filename_size) +{ + snprintf(full_filename, filename_size, \ + "%s/data/"SYNC_DIR_NAME"/%s_%d%s", g_fdfs_base_path, \ + ip_addr, port, SYNC_MARK_FILE_EXT); + return full_filename; +} + +char *get_mark_filename_by_reader(const void *pArg, char *full_filename) +{ + const StorageBinLogReader *pReader; + static char buff[MAX_PATH_SIZE]; + + pReader = (const StorageBinLogReader *)pArg; + if (full_filename == NULL) + { + full_filename = buff; + } + + return get_mark_filename_by_id_and_port(pReader->storage_id, \ + g_server_port, full_filename, MAX_PATH_SIZE); +} + +static char *get_mark_filename_by_id(const char *storage_id, \ + char *full_filename, const int filename_size) +{ + return get_mark_filename_by_id_and_port(storage_id, g_server_port, \ + full_filename, filename_size); +} + +int storage_report_storage_status(const char *storage_id, \ + const char *ip_addr, const char status) +{ + FDFSStorageBrief briefServer; + ConnectionInfo trackerServer; + ConnectionInfo *pGlobalServer; + ConnectionInfo *pTServer; + ConnectionInfo *pTServerEnd; + int result; + int report_count; + int success_count; + int i; + + memset(&briefServer, 0, sizeof(FDFSStorageBrief)); + strcpy(briefServer.id, storage_id); + strcpy(briefServer.ip_addr, ip_addr); + briefServer.status = status; + + logDebug("file: "__FILE__", line: %d, " \ + "begin to report storage %s 's status as: %d", \ + __LINE__, ip_addr, status); + + if (!g_sync_old_done) + { + logDebug("file: "__FILE__", line: %d, " \ + "report storage %s 's status as: %d, " \ + "waiting for g_sync_old_done turn to true...", \ + __LINE__, ip_addr, status); + + while (g_continue_flag && !g_sync_old_done) + { + sleep(1); + } + + if (!g_continue_flag) + { + return 0; + } + + logDebug("file: "__FILE__", line: %d, " \ + "report storage %s 's status as: %d, " \ + "ok, g_sync_old_done turn to true", \ + __LINE__, ip_addr, status); + } + + report_count = 0; + success_count = 0; + + result = 0; + pTServer = &trackerServer; + pTServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + for (pGlobalServer=g_tracker_group.servers; pGlobalServersock = socket(AF_INET, SOCK_STREAM, 0); + if(pTServer->sock < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s.", \ + __LINE__, result, STRERROR(result)); + sleep(5); + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(pTServer->sock, g_bind_addr, 0); + } + + tcpsetserveropt(pTServer->sock, g_fdfs_network_timeout); + + if (tcpsetnonblockopt(pTServer->sock) != 0) + { + close(pTServer->sock); + pTServer->sock = -1; + sleep(5); + continue; + } + + if ((result=connectserverbyip_nb(pTServer->sock, \ + pTServer->ip_addr, pTServer->port, \ + g_fdfs_connect_timeout)) == 0) + { + break; + } + + close(pTServer->sock); + pTServer->sock = -1; + sleep(5); + } + + if (pTServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTServer->ip_addr, pTServer->port, \ + result, STRERROR(result)); + + continue; + } + + report_count++; + if ((result=tracker_report_storage_status(pTServer, \ + &briefServer)) == 0) + { + success_count++; + } + + fdfs_quit(pTServer); + close(pTServer->sock); + } + + logDebug("file: "__FILE__", line: %d, " \ + "report storage %s 's status as: %d done, " \ + "report count: %d, success count: %d", \ + __LINE__, ip_addr, status, report_count, success_count); + + return success_count > 0 ? 0 : EAGAIN; +} + +static int storage_reader_sync_init_req(StorageBinLogReader *pReader) +{ + ConnectionInfo *pTrackerServers; + ConnectionInfo *pTServer; + ConnectionInfo *pTServerEnd; + char tracker_client_ip[IP_ADDRESS_SIZE]; + int result; + int conn_ret; + + if (!g_sync_old_done) + { + while (g_continue_flag && !g_sync_old_done) + { + sleep(1); + } + + if (!g_continue_flag) + { + return EINTR; + } + } + + pTrackerServers = (ConnectionInfo *)malloc( \ + sizeof(ConnectionInfo) * g_tracker_group.server_count); + if (pTrackerServers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(ConnectionInfo) * \ + g_tracker_group.server_count); + return errno != 0 ? errno : ENOMEM; + } + + memcpy(pTrackerServers, g_tracker_group.servers, \ + sizeof(ConnectionInfo) * g_tracker_group.server_count); + pTServerEnd = pTrackerServers + g_tracker_group.server_count; + for (pTServer=pTrackerServers; pTServersock = -1; + } + + result = EINTR; + if (g_tracker_group.leader_index >= 0 && \ + g_tracker_group.leader_index < g_tracker_group.server_count) + { + pTServer = pTrackerServers + g_tracker_group.leader_index; + } + else + { + pTServer = pTrackerServers; + } + do + { + while (g_continue_flag) + { + pTServer->sock = socket(AF_INET, SOCK_STREAM, 0); + if(pTServer->sock < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s. program exit!", \ + __LINE__, errno, STRERROR(errno)); + g_continue_flag = false; + result = errno != 0 ? errno : EPERM; + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(pTServer->sock, g_bind_addr, 0); + } + + if (tcpsetnonblockopt(pTServer->sock) != 0) + { + close(pTServer->sock); + sleep(g_heart_beat_interval); + continue; + } + + if ((conn_ret=connectserverbyip_nb(pTServer->sock, \ + pTServer->ip_addr, pTServer->port, \ + g_fdfs_connect_timeout)) == 0) + { + break; + } + + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTServer->ip_addr, pTServer->port, \ + conn_ret, STRERROR(conn_ret)); + + close(pTServer->sock); + + pTServer++; + if (pTServer >= pTServerEnd) + { + pTServer = pTrackerServers; + } + + sleep(g_heart_beat_interval); + } + + if (!g_continue_flag) + { + break; + } + + getSockIpaddr(pTServer->sock, \ + tracker_client_ip, IP_ADDRESS_SIZE); + insert_into_local_host_ip(tracker_client_ip); + + if ((result=tracker_sync_src_req(pTServer, pReader)) != 0) + { + fdfs_quit(pTServer); + close(pTServer->sock); + sleep(g_heart_beat_interval); + continue; + } + + fdfs_quit(pTServer); + close(pTServer->sock); + + break; + } while (1); + + free(pTrackerServers); + + /* + //printf("need_sync_old=%d, until_timestamp=%d\n", \ + pReader->need_sync_old, pReader->until_timestamp); + */ + + return result; +} + +int storage_reader_init(FDFSStorageBrief *pStorage, StorageBinLogReader *pReader) +{ + char full_filename[MAX_PATH_SIZE]; + IniContext iniContext; + int result; + bool bFileExist; + bool bNeedSyncOld; + + memset(pReader, 0, sizeof(StorageBinLogReader)); + pReader->mark_fd = -1; + pReader->binlog_fd = -1; + + pReader->binlog_buff.buffer = (char *)malloc( \ + STORAGE_BINLOG_BUFFER_SIZE); + if (pReader->binlog_buff.buffer == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, STORAGE_BINLOG_BUFFER_SIZE, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + pReader->binlog_buff.current = pReader->binlog_buff.buffer; + + if (pStorage == NULL) + { + strcpy(pReader->storage_id, "0.0.0.0"); + } + else + { + strcpy(pReader->storage_id, pStorage->id); + } + get_mark_filename_by_reader(pReader, full_filename); + + if (pStorage == NULL) + { + bFileExist = false; + } + else if (pStorage->status <= FDFS_STORAGE_STATUS_WAIT_SYNC) + { + bFileExist = false; + } + else + { + bFileExist = fileExists(full_filename); + if (!bFileExist && (g_use_storage_id && pStorage != NULL)) + { + char old_mark_filename[MAX_PATH_SIZE]; + get_mark_filename_by_ip_and_port(pStorage->ip_addr, \ + g_server_port, old_mark_filename, \ + sizeof(old_mark_filename)); + if (fileExists(old_mark_filename)) + { + if (rename(old_mark_filename, full_filename)!=0) + { + logError("file: "__FILE__", line: %d, "\ + "rename file %s to %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, old_mark_filename, \ + full_filename, errno, \ + STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + bFileExist = true; + } + } + } + + if (pStorage != NULL && !bFileExist) + { + if ((result=storage_reader_sync_init_req(pReader)) != 0) + { + return result; + } + } + + if (bFileExist) + { + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(full_filename, &iniContext)) \ + != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load from mark file \"%s\" fail, " \ + "error code: %d", \ + __LINE__, full_filename, result); + return result; + } + + if (iniContext.global.count < 7) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", item count: %d < 7", \ + __LINE__, full_filename, iniContext.global.count); + return ENOENT; + } + + bNeedSyncOld = iniGetBoolValue(NULL, \ + MARK_ITEM_NEED_SYNC_OLD, \ + &iniContext, false); + if (pStorage != NULL && pStorage->status == \ + FDFS_STORAGE_STATUS_SYNCING) + { + if ((result=storage_reader_sync_init_req(pReader)) != 0) + { + iniFreeContext(&iniContext); + return result; + } + + if (pReader->need_sync_old && !bNeedSyncOld) + { + bFileExist = false; //re-sync + } + else + { + pReader->need_sync_old = bNeedSyncOld; + } + } + else + { + pReader->need_sync_old = bNeedSyncOld; + } + + if (bFileExist) + { + pReader->binlog_index = iniGetIntValue(NULL, \ + MARK_ITEM_BINLOG_FILE_INDEX, \ + &iniContext, -1); + pReader->binlog_offset = iniGetInt64Value(NULL, \ + MARK_ITEM_BINLOG_FILE_OFFSET, \ + &iniContext, -1); + pReader->sync_old_done = iniGetBoolValue(NULL, \ + MARK_ITEM_SYNC_OLD_DONE, \ + &iniContext, false); + pReader->until_timestamp = iniGetIntValue(NULL, \ + MARK_ITEM_UNTIL_TIMESTAMP, \ + &iniContext, -1); + pReader->scan_row_count = iniGetInt64Value(NULL, \ + MARK_ITEM_SCAN_ROW_COUNT, \ + &iniContext, 0); + pReader->sync_row_count = iniGetInt64Value(NULL, \ + MARK_ITEM_SYNC_ROW_COUNT, \ + &iniContext, 0); + + if (pReader->binlog_index < 0) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", " \ + "binlog_index: %d < 0", \ + __LINE__, full_filename, \ + pReader->binlog_index); + return EINVAL; + } + if (pReader->binlog_offset < 0) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", binlog_offset: "\ + INT64_PRINTF_FORMAT" < 0", \ + __LINE__, full_filename, \ + pReader->binlog_offset); + return EINVAL; + } + } + + iniFreeContext(&iniContext); + } + + pReader->last_scan_rows = pReader->scan_row_count; + pReader->last_sync_rows = pReader->sync_row_count; + + pReader->mark_fd = open(full_filename, O_WRONLY | O_CREAT, 0644); + if (pReader->mark_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open mark file \"%s\" fail, " \ + "error no: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + STORAGE_FCHOWN(pReader->mark_fd, full_filename, geteuid(), getegid()) + + if ((result=storage_open_readable_binlog(pReader, \ + get_binlog_readable_filename, pReader)) != 0) + { + return result; + } + + if (pStorage != NULL && !bFileExist) + { + if (!pReader->need_sync_old && pReader->until_timestamp > 0) + { + if ((result=storage_binlog_reader_skip(pReader)) != 0) + { + return result; + } + } + + if ((result=storage_write_to_mark_file(pReader)) != 0) + { + return result; + } + } + + result = storage_binlog_preread(pReader); + if (result != 0 && result != ENOENT) + { + return result; + } + + return 0; +} + +void storage_reader_destroy(StorageBinLogReader *pReader) +{ + if (pReader->mark_fd >= 0) + { + close(pReader->mark_fd); + pReader->mark_fd = -1; + } + + if (pReader->binlog_fd >= 0) + { + close(pReader->binlog_fd); + pReader->binlog_fd = -1; + } + + if (pReader->binlog_buff.buffer != NULL) + { + free(pReader->binlog_buff.buffer); + pReader->binlog_buff.buffer = NULL; + pReader->binlog_buff.current = NULL; + pReader->binlog_buff.length = 0; + } +} + +static int storage_write_to_mark_file(StorageBinLogReader *pReader) +{ + char buff[256]; + int len; + int result; + + len = sprintf(buff, \ + "%s=%d\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s=%d\n" \ + "%s="INT64_PRINTF_FORMAT"\n" \ + "%s="INT64_PRINTF_FORMAT"\n", \ + MARK_ITEM_BINLOG_FILE_INDEX, pReader->binlog_index, \ + MARK_ITEM_BINLOG_FILE_OFFSET, pReader->binlog_offset, \ + MARK_ITEM_NEED_SYNC_OLD, pReader->need_sync_old, \ + MARK_ITEM_SYNC_OLD_DONE, pReader->sync_old_done, \ + MARK_ITEM_UNTIL_TIMESTAMP, (int)pReader->until_timestamp, \ + MARK_ITEM_SCAN_ROW_COUNT, pReader->scan_row_count, \ + MARK_ITEM_SYNC_ROW_COUNT, pReader->sync_row_count); + + if ((result=storage_write_to_fd(pReader->mark_fd, \ + get_mark_filename_by_reader, pReader, buff, len)) == 0) + { + pReader->last_scan_rows = pReader->scan_row_count; + pReader->last_sync_rows = pReader->sync_row_count; + } + + return result; +} + +static int rewind_to_prev_rec_end(StorageBinLogReader *pReader) +{ + if (lseek(pReader->binlog_fd, pReader->binlog_offset, SEEK_SET) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "seek binlog file \"%s\"fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "errno: %d, error info: %s", \ + __LINE__, get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + pReader->binlog_buff.current = pReader->binlog_buff.buffer; + pReader->binlog_buff.length = 0; + + return 0; +} + +static int storage_binlog_preread(StorageBinLogReader *pReader) +{ + int bytes_read; + int saved_binlog_write_version; + + if (pReader->binlog_buff.version == binlog_write_version && \ + pReader->binlog_buff.length == 0) + { + return ENOENT; + } + + saved_binlog_write_version = binlog_write_version; + if (pReader->binlog_buff.current != pReader->binlog_buff.buffer) + { + if (pReader->binlog_buff.length > 0) + { + memcpy(pReader->binlog_buff.buffer, \ + pReader->binlog_buff.current, \ + pReader->binlog_buff.length); + } + + pReader->binlog_buff.current = pReader->binlog_buff.buffer; + } + + bytes_read = read(pReader->binlog_fd, pReader->binlog_buff.buffer \ + + pReader->binlog_buff.length, \ + STORAGE_BINLOG_BUFFER_SIZE - pReader->binlog_buff.length); + if (bytes_read < 0) + { + logError("file: "__FILE__", line: %d, " \ + "read from binlog file \"%s\" fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "error no: %d, error info: %s", __LINE__, \ + get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset + pReader->binlog_buff.length, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + else if (bytes_read == 0) //end of binlog file + { + pReader->binlog_buff.version = saved_binlog_write_version; + return ENOENT; + } + + pReader->binlog_buff.length += bytes_read; + return 0; +} + +static int storage_binlog_do_line_read(StorageBinLogReader *pReader, \ + char *line, const int line_size, int *line_length) +{ + char *pLineEnd; + + if (pReader->binlog_buff.length == 0) + { + *line_length = 0; + return ENOENT; + } + + pLineEnd = (char *)memchr(pReader->binlog_buff.current, '\n', \ + pReader->binlog_buff.length); + if (pLineEnd == NULL) + { + *line_length = 0; + return ENOENT; + } + + *line_length = (pLineEnd - pReader->binlog_buff.current) + 1; + if (*line_length >= line_size) + { + logError("file: "__FILE__", line: %d, " \ + "read from binlog file \"%s\" fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "line buffer size: %d is too small! " \ + "<= line length: %d", __LINE__, \ + get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset, line_size, *line_length); + return ENOSPC; + } + + memcpy(line, pReader->binlog_buff.current, *line_length); + *(line + *line_length) = '\0'; + + pReader->binlog_buff.current = pLineEnd + 1; + pReader->binlog_buff.length -= *line_length; + + return 0; +} + +static int storage_binlog_read_line(StorageBinLogReader *pReader, \ + char *line, const int line_size, int *line_length) +{ + int result; + + result = storage_binlog_do_line_read(pReader, line, \ + line_size, line_length); + if (result != ENOENT) + { + return result; + } + + result = storage_binlog_preread(pReader); + if (result != 0) + { + return result; + } + + return storage_binlog_do_line_read(pReader, line, \ + line_size, line_length); +} + +int storage_binlog_read(StorageBinLogReader *pReader, \ + StorageBinLogRecord *pRecord, int *record_length) +{ + char line[STORAGE_BINLOG_LINE_SIZE]; + char *cols[3]; + int result; + + while (1) + { + result = storage_binlog_read_line(pReader, line, \ + sizeof(line), record_length); + if (result == 0) + { + break; + } + else if (result != ENOENT) + { + return result; + } + + if (pReader->binlog_index >= g_binlog_index) + { + return ENOENT; + } + + if (pReader->binlog_buff.length != 0) + { + logError("file: "__FILE__", line: %d, " \ + "binlog file \"%s\" not ended by \\n, " \ + "file offset: "INT64_PRINTF_FORMAT, __LINE__, \ + get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset); + return ENOENT; + } + + //rotate + pReader->binlog_index++; + pReader->binlog_offset = 0; + pReader->binlog_buff.version = 0; + if ((result=storage_open_readable_binlog(pReader, \ + get_binlog_readable_filename, pReader)) != 0) + { + return result; + } + + if ((result=storage_write_to_mark_file(pReader)) != 0) + { + return result; + } + } + + if ((result=splitEx(line, ' ', cols, 3)) < 3) + { + logError("file: "__FILE__", line: %d, " \ + "read data from binlog file \"%s\" fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "read item count: %d < 3", \ + __LINE__, get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset, result); + return EINVAL; + } + + pRecord->timestamp = atoi(cols[0]); + pRecord->op_type = *(cols[1]); + pRecord->filename_len = strlen(cols[2]) - 1; //need trim new line \n + if (pRecord->filename_len > sizeof(pRecord->filename) - 1) + { + logError("file: "__FILE__", line: %d, " \ + "item \"filename\" in binlog " \ + "file \"%s\" is invalid, file offset: " \ + INT64_PRINTF_FORMAT", filename length: %d > %d", \ + __LINE__, get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset, \ + pRecord->filename_len, (int)sizeof(pRecord->filename)-1); + return EINVAL; + } + + memcpy(pRecord->filename, cols[2], pRecord->filename_len); + *(pRecord->filename + pRecord->filename_len) = '\0'; + if (pRecord->op_type == STORAGE_OP_TYPE_SOURCE_CREATE_LINK || \ + pRecord->op_type == STORAGE_OP_TYPE_REPLICA_CREATE_LINK) + { + char *p; + + p = strchr(pRecord->filename, ' '); + if (p == NULL) + { + *(pRecord->src_filename) = '\0'; + pRecord->src_filename_len = 0; + } + else + { + pRecord->src_filename_len = pRecord->filename_len - \ + (p - pRecord->filename) - 1; + pRecord->filename_len = p - pRecord->filename; + *p = '\0'; + + memcpy(pRecord->src_filename, p + 1, \ + pRecord->src_filename_len); + *(pRecord->src_filename + \ + pRecord->src_filename_len) = '\0'; + } + } + else + { + *(pRecord->src_filename) = '\0'; + pRecord->src_filename_len = 0; + } + + pRecord->true_filename_len = pRecord->filename_len; + if ((result=storage_split_filename_ex(pRecord->filename, \ + &pRecord->true_filename_len, pRecord->true_filename, \ + &pRecord->store_path_index)) != 0) + { + return result; + } + + /* + //printf("timestamp=%d, op_type=%c, filename=%s(%d), line length=%d, " \ + "offset=%d\n", \ + pRecord->timestamp, pRecord->op_type, \ + pRecord->filename, strlen(pRecord->filename), \ + *record_length, pReader->binlog_offset); + */ + + return 0; +} + +static int storage_binlog_reader_skip(StorageBinLogReader *pReader) +{ + StorageBinLogRecord record; + int result; + int record_len; + + while (1) + { + result = storage_binlog_read(pReader, \ + &record, &record_len); + if (result != 0) + { + if (result == ENOENT) + { + return 0; + } + + if (result == EINVAL && g_file_sync_skip_invalid_record) + { + logWarning("file: "__FILE__", line: %d, " \ + "skip invalid record!", __LINE__); + } + else + { + return result; + } + } + + if (record.timestamp >= pReader->until_timestamp) + { + result = rewind_to_prev_rec_end(pReader); + break; + } + + pReader->binlog_offset += record_len; + } + + return result; +} + +int storage_unlink_mark_file(const char *storage_id) +{ + char old_filename[MAX_PATH_SIZE]; + char new_filename[MAX_PATH_SIZE]; + time_t t; + struct tm tm; + + t = g_current_time; + localtime_r(&t, &tm); + + get_mark_filename_by_id(storage_id, old_filename, sizeof(old_filename)); + if (!fileExists(old_filename)) + { + return ENOENT; + } + + snprintf(new_filename, sizeof(new_filename), \ + "%s.%04d%02d%02d%02d%02d%02d", old_filename, \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec); + if (rename(old_filename, new_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file %s to %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, old_filename, new_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +int storage_rename_mark_file(const char *old_ip_addr, const int old_port, \ + const char *new_ip_addr, const int new_port) +{ + char old_filename[MAX_PATH_SIZE]; + char new_filename[MAX_PATH_SIZE]; + + get_mark_filename_by_id_and_port(old_ip_addr, old_port, \ + old_filename, sizeof(old_filename)); + if (!fileExists(old_filename)) + { + return ENOENT; + } + + get_mark_filename_by_id_and_port(new_ip_addr, new_port, \ + new_filename, sizeof(new_filename)); + if (fileExists(new_filename)) + { + logWarning("file: "__FILE__", line: %d, " \ + "mark file %s already exists, " \ + "ignore rename file %s to %s", \ + __LINE__, new_filename, old_filename, new_filename); + return EEXIST; + } + + if (rename(old_filename, new_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file %s to %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, old_filename, new_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +static void storage_sync_get_start_end_times(time_t current_time, \ + const TimeInfo *pStartTime, const TimeInfo *pEndTime, \ + time_t *start_time, time_t *end_time) +{ + struct tm tm_time; + //char buff[32]; + + localtime_r(¤t_time, &tm_time); + tm_time.tm_sec = 0; + + /* + strftime(buff, sizeof(buff), "%Y-%m-%d %H:%M:%S", &tm_time); + //printf("current time: %s\n", buff); + */ + + tm_time.tm_hour = pStartTime->hour; + tm_time.tm_min = pStartTime->minute; + *start_time = mktime(&tm_time); + + //end time < start time + if (pEndTime->hour < pStartTime->hour || (pEndTime->hour == \ + pStartTime->hour && pEndTime->minute < pStartTime->minute)) + { + current_time += 24 * 3600; + localtime_r(¤t_time, &tm_time); + tm_time.tm_sec = 0; + } + + tm_time.tm_hour = pEndTime->hour; + tm_time.tm_min = pEndTime->minute; + *end_time = mktime(&tm_time); +} + +static void storage_sync_thread_exit(ConnectionInfo *pStorage) +{ + int result; + int i; + pthread_t tid; + + if ((result=pthread_mutex_lock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + tid = pthread_self(); + for (i=0; iip_addr, pStorage->port); +} + +static void* storage_sync_thread_entrance(void* arg) +{ + FDFSStorageBrief *pStorage; + StorageBinLogReader reader; + StorageBinLogRecord record; + ConnectionInfo storage_server; + char local_ip_addr[IP_ADDRESS_SIZE]; + int read_result; + int sync_result; + int conn_result; + int result; + int record_len; + int previousCode; + int nContinuousFail; + time_t current_time; + time_t start_time; + time_t end_time; + time_t last_keep_alive_time; + + memset(local_ip_addr, 0, sizeof(local_ip_addr)); + memset(&reader, 0, sizeof(reader)); + reader.mark_fd = -1; + reader.binlog_fd = -1; + + current_time = g_current_time; + last_keep_alive_time = 0; + start_time = 0; + end_time = 0; + + pStorage = (FDFSStorageBrief *)arg; + + strcpy(storage_server.ip_addr, pStorage->ip_addr); + storage_server.port = g_server_port; + storage_server.sock = -1; + + logDebug("file: "__FILE__", line: %d, " \ + "sync thread to storage server %s:%d started", \ + __LINE__, storage_server.ip_addr, storage_server.port); + + while (g_continue_flag && \ + pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ + pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ + pStorage->status != FDFS_STORAGE_STATUS_NONE) + { + while (g_continue_flag && \ + (pStorage->status == FDFS_STORAGE_STATUS_INIT || + pStorage->status == FDFS_STORAGE_STATUS_OFFLINE || + pStorage->status == FDFS_STORAGE_STATUS_ONLINE)) + { + sleep(1); + } + + if ((!g_continue_flag) || + pStorage->status == FDFS_STORAGE_STATUS_DELETED || \ + pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + pStorage->status == FDFS_STORAGE_STATUS_NONE) + { + break; + } + + if (g_sync_part_time) + { + current_time = g_current_time; + storage_sync_get_start_end_times(current_time, \ + &g_sync_end_time, &g_sync_start_time, \ + &start_time, &end_time); + start_time += 60; + end_time -= 60; + while (g_continue_flag && (current_time >= start_time \ + && current_time <= end_time)) + { + current_time = g_current_time; + sleep(1); + } + } + + previousCode = 0; + nContinuousFail = 0; + conn_result = 0; + while (g_continue_flag && \ + pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ + pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ + pStorage->status != FDFS_STORAGE_STATUS_NONE) + { + strcpy(storage_server.ip_addr, pStorage->ip_addr); + storage_server.sock = \ + socket(AF_INET, SOCK_STREAM, 0); + if(storage_server.sock < 0) + { + logCrit("file: "__FILE__", line: %d," \ + " socket create fail, " \ + "errno: %d, error info: %s. " \ + "program exit!", __LINE__, \ + errno, STRERROR(errno)); + g_continue_flag = false; + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(storage_server.sock, g_bind_addr, 0); + } + + if (tcpsetnonblockopt(storage_server.sock) != 0) + { + nContinuousFail++; + close(storage_server.sock); + storage_server.sock = -1; + sleep(1); + + continue; + } + + if ((conn_result=connectserverbyip_nb(storage_server.sock,\ + pStorage->ip_addr, g_server_port, \ + g_fdfs_connect_timeout)) == 0) + { + char szFailPrompt[64]; + if (nContinuousFail == 0) + { + *szFailPrompt = '\0'; + } + else + { + sprintf(szFailPrompt, \ + ", continuous fail count: %d", \ + nContinuousFail); + } + logInfo("file: "__FILE__", line: %d, " \ + "successfully connect to " \ + "storage server %s:%d%s", __LINE__, \ + pStorage->ip_addr, \ + g_server_port, szFailPrompt); + nContinuousFail = 0; + break; + } + + if (previousCode != conn_result) + { + logError("file: "__FILE__", line: %d, " \ + "connect to storage server %s:%d fail" \ + ", errno: %d, error info: %s", \ + __LINE__, \ + pStorage->ip_addr, g_server_port, \ + conn_result, STRERROR(conn_result)); + previousCode = conn_result; + } + + nContinuousFail++; + close(storage_server.sock); + storage_server.sock = -1; + + if (!g_continue_flag) + { + break; + } + + sleep(1); + } + + if (nContinuousFail > 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to storage server %s:%d fail, " \ + "try count: %d, errno: %d, error info: %s", \ + __LINE__, pStorage->ip_addr, \ + g_server_port, nContinuousFail, \ + conn_result, STRERROR(conn_result)); + } + + if ((!g_continue_flag) || + pStorage->status == FDFS_STORAGE_STATUS_DELETED || \ + pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + pStorage->status == FDFS_STORAGE_STATUS_NONE) + { + break; + } + + if (pStorage->status != FDFS_STORAGE_STATUS_ACTIVE && \ + pStorage->status != FDFS_STORAGE_STATUS_WAIT_SYNC && \ + pStorage->status != FDFS_STORAGE_STATUS_SYNCING) + { + close(storage_server.sock); + sleep(5); + continue; + } + + if ((result=storage_reader_init(pStorage, &reader)) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_reader_init fail, errno=%d, " \ + "program exit!", \ + __LINE__, result); + g_continue_flag = false; + break; + } + + if (!reader.need_sync_old) + { + while (g_continue_flag && \ + (pStorage->status != FDFS_STORAGE_STATUS_ACTIVE && \ + pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ + pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ + pStorage->status != FDFS_STORAGE_STATUS_NONE)) + { + sleep(1); + } + + if (pStorage->status != FDFS_STORAGE_STATUS_ACTIVE) + { + close(storage_server.sock); + storage_reader_destroy(&reader); + continue; + } + } + + getSockIpaddr(storage_server.sock, \ + local_ip_addr, IP_ADDRESS_SIZE); + insert_into_local_host_ip(local_ip_addr); + + /* + //printf("file: "__FILE__", line: %d, " \ + "storage_server.ip_addr=%s, " \ + "local_ip_addr: %s\n", \ + __LINE__, pStorage->ip_addr, local_ip_addr); + */ + + if (is_local_host_ip(pStorage->ip_addr)) + { //can't self sync to self + logError("file: "__FILE__", line: %d, " \ + "ip_addr %s belong to the local host," \ + " sync thread exit.", \ + __LINE__, pStorage->ip_addr); + fdfs_quit(&storage_server); + close(storage_server.sock); + break; + } + + if (storage_report_my_server_id(&storage_server) != 0) + { + close(storage_server.sock); + storage_reader_destroy(&reader); + sleep(1); + continue; + } + + if (pStorage->status == FDFS_STORAGE_STATUS_WAIT_SYNC) + { + pStorage->status = FDFS_STORAGE_STATUS_SYNCING; + storage_report_storage_status(pStorage->id, \ + pStorage->ip_addr, pStorage->status); + } + + if (pStorage->status == FDFS_STORAGE_STATUS_SYNCING) + { + if (reader.need_sync_old && reader.sync_old_done) + { + pStorage->status = FDFS_STORAGE_STATUS_OFFLINE; + storage_report_storage_status(pStorage->id, \ + pStorage->ip_addr, \ + pStorage->status); + } + } + + if (g_sync_part_time) + { + current_time = g_current_time; + storage_sync_get_start_end_times(current_time, \ + &g_sync_start_time, &g_sync_end_time, \ + &start_time, &end_time); + } + + sync_result = 0; + while (g_continue_flag && (!g_sync_part_time || \ + (current_time >= start_time && \ + current_time <= end_time)) && \ + (pStorage->status == FDFS_STORAGE_STATUS_ACTIVE || \ + pStorage->status == FDFS_STORAGE_STATUS_SYNCING)) + { + read_result = storage_binlog_read(&reader, \ + &record, &record_len); + if (read_result == ENOENT) + { + if (reader.need_sync_old && \ + !reader.sync_old_done) + { + reader.sync_old_done = true; + if (storage_write_to_mark_file(&reader) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_write_to_mark_file " \ + "fail, program exit!", \ + __LINE__); + g_continue_flag = false; + break; + } + + if (pStorage->status == \ + FDFS_STORAGE_STATUS_SYNCING) + { + pStorage->status = \ + FDFS_STORAGE_STATUS_OFFLINE; + storage_report_storage_status( \ + pStorage->id, \ + pStorage->ip_addr, \ + pStorage->status); + } + } + + + if (reader.last_scan_rows!=reader.scan_row_count) + { + if (storage_write_to_mark_file(&reader)!=0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_write_to_mark_file fail, " \ + "program exit!", __LINE__); + g_continue_flag = false; + break; + } + } + + current_time = g_current_time; + if (current_time - last_keep_alive_time >= \ + g_heart_beat_interval) + { + if (fdfs_active_test(&storage_server)!=0) + { + break; + } + + last_keep_alive_time = current_time; + } + + usleep(g_sync_wait_usec); + continue; + } + + if (g_sync_part_time) + { + current_time = g_current_time; + } + + if (read_result != 0) + { + if (read_result == EINVAL && \ + g_file_sync_skip_invalid_record) + { + logWarning("file: "__FILE__", line: %d, " \ + "skip invalid record, binlog index: " \ + "%d, offset: "INT64_PRINTF_FORMAT, \ + __LINE__, reader.binlog_index, \ + reader.binlog_offset); + } + else + { + sleep(5); + break; + } + } + else if ((sync_result=storage_sync_data(&reader, \ + &storage_server, &record)) != 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "binlog index: %d, current record " \ + "offset: "INT64_PRINTF_FORMAT", next " \ + "record offset: "INT64_PRINTF_FORMAT, \ + __LINE__, reader.binlog_index, \ + reader.binlog_offset, \ + reader.binlog_offset + record_len); + if (rewind_to_prev_rec_end(&reader) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "rewind_to_prev_rec_end fail, "\ + "program exit!", __LINE__); + g_continue_flag = false; + } + + break; + } + + reader.binlog_offset += record_len; + reader.scan_row_count++; + + if (g_sync_interval > 0) + { + usleep(g_sync_interval); + } + } + + if (reader.last_scan_rows != reader.scan_row_count) + { + if (storage_write_to_mark_file(&reader) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_write_to_mark_file fail, " \ + "program exit!", __LINE__); + g_continue_flag = false; + break; + } + } + + close(storage_server.sock); + storage_server.sock = -1; + storage_reader_destroy(&reader); + + if (!g_continue_flag) + { + break; + } + + if (!(sync_result == ENOTCONN || sync_result == EIO)) + { + sleep(1); + } + } + + if (storage_server.sock >= 0) + { + close(storage_server.sock); + } + storage_reader_destroy(&reader); + + if (pStorage->status == FDFS_STORAGE_STATUS_DELETED + || pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED) + { + storage_changelog_req(); + sleep(2 * g_heart_beat_interval + 1); + pStorage->status = FDFS_STORAGE_STATUS_NONE; + } + + storage_sync_thread_exit(&storage_server); + + return NULL; +} + +int storage_sync_thread_start(const FDFSStorageBrief *pStorage) +{ + int result; + pthread_attr_t pattr; + pthread_t tid; + + if (pStorage->status == FDFS_STORAGE_STATUS_DELETED || \ + pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + pStorage->status == FDFS_STORAGE_STATUS_NONE) + { + logWarning("file: "__FILE__", line: %d, " \ + "storage id: %s 's status: %d is invalid, " \ + "can't start sync thread!", __LINE__, \ + pStorage->id, pStorage->status); + return 0; + } + + if (storage_server_is_myself(pStorage) || \ + is_local_host_ip(pStorage->ip_addr)) //can't self sync to self + { + logWarning("file: "__FILE__", line: %d, " \ + "storage id: %s is myself, can't start sync thread!", \ + __LINE__, pStorage->id); + return 0; + } + + if ((result=init_pthread_attr(&pattr, g_thread_stack_size)) != 0) + { + return result; + } + + /* + //printf("start storage ip_addr: %s, g_storage_sync_thread_count=%d\n", + pStorage->ip_addr, g_storage_sync_thread_count); + */ + + if ((result=pthread_create(&tid, &pattr, storage_sync_thread_entrance, \ + (void *)pStorage)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "create thread failed, errno: %d, " \ + "error info: %s", \ + __LINE__, result, STRERROR(result)); + + pthread_attr_destroy(&pattr); + return result; + } + + if ((result=pthread_mutex_lock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + g_storage_sync_thread_count++; + sync_tids = (pthread_t *)realloc(sync_tids, sizeof(pthread_t) * \ + g_storage_sync_thread_count); + if (sync_tids == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(pthread_t) * \ + g_storage_sync_thread_count, \ + errno, STRERROR(errno)); + } + else + { + sync_tids[g_storage_sync_thread_count - 1] = tid; + } + + if ((result=pthread_mutex_unlock(&sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + pthread_attr_destroy(&pattr); + + return 0; +} + diff --git a/storage/storage_sync.h b/storage/storage_sync.h new file mode 100644 index 0000000..6ca73f8 --- /dev/null +++ b/storage/storage_sync.h @@ -0,0 +1,109 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//storage_sync.h + +#ifndef _STORAGE_SYNC_H_ +#define _STORAGE_SYNC_H_ + +#include "storage_func.h" + +#define STORAGE_OP_TYPE_SOURCE_CREATE_FILE 'C' //upload file +#define STORAGE_OP_TYPE_SOURCE_APPEND_FILE 'A' //append file +#define STORAGE_OP_TYPE_SOURCE_DELETE_FILE 'D' //delete file +#define STORAGE_OP_TYPE_SOURCE_UPDATE_FILE 'U' //for whole file update such as metadata file +#define STORAGE_OP_TYPE_SOURCE_MODIFY_FILE 'M' //for part modify +#define STORAGE_OP_TYPE_SOURCE_TRUNCATE_FILE 'T' //truncate file +#define STORAGE_OP_TYPE_SOURCE_CREATE_LINK 'L' //create symbol link +#define STORAGE_OP_TYPE_REPLICA_CREATE_FILE 'c' +#define STORAGE_OP_TYPE_REPLICA_APPEND_FILE 'a' +#define STORAGE_OP_TYPE_REPLICA_DELETE_FILE 'd' +#define STORAGE_OP_TYPE_REPLICA_UPDATE_FILE 'u' +#define STORAGE_OP_TYPE_REPLICA_MODIFY_FILE 'm' +#define STORAGE_OP_TYPE_REPLICA_TRUNCATE_FILE 't' +#define STORAGE_OP_TYPE_REPLICA_CREATE_LINK 'l' + +#define STORAGE_BINLOG_BUFFER_SIZE 64 * 1024 +#define STORAGE_BINLOG_LINE_SIZE 256 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct +{ + char storage_id[FDFS_STORAGE_ID_MAX_SIZE]; + bool need_sync_old; + bool sync_old_done; + bool last_file_exist; //if the last file exist on the dest server + BinLogBuffer binlog_buff; + time_t until_timestamp; + int mark_fd; + int binlog_index; + int binlog_fd; + int64_t binlog_offset; + int64_t scan_row_count; + int64_t sync_row_count; + + int64_t last_scan_rows; //for write to mark file + int64_t last_sync_rows; //for write to mark file +} StorageBinLogReader; + +typedef struct +{ + time_t timestamp; + char op_type; + char filename[128]; //filename with path index prefix which should be trimed + char true_filename[128]; //pure filename + char src_filename[128]; //src filename with path index prefix + int filename_len; + int true_filename_len; + int src_filename_len; + int store_path_index; +} StorageBinLogRecord; + +extern int g_binlog_fd; +extern int g_binlog_index; + +extern int g_storage_sync_thread_count; + +int storage_sync_init(); +int storage_sync_destroy(); + +#define storage_binlog_write(timestamp, op_type, filename) \ + storage_binlog_write_ex(timestamp, op_type, filename, NULL) + +int storage_binlog_write_ex(const int timestamp, const char op_type, \ + const char *filename, const char *extra); + +int storage_binlog_read(StorageBinLogReader *pReader, \ + StorageBinLogRecord *pRecord, int *record_length); + +int storage_sync_thread_start(const FDFSStorageBrief *pStorage); +int kill_storage_sync_threads(); +int fdfs_binlog_sync_func(void *args); + +char *get_mark_filename_by_reader(const void *pArg, char *full_filename); +int storage_unlink_mark_file(const char *storage_id); +int storage_rename_mark_file(const char *old_ip_addr, const int old_port, \ + const char *new_ip_addr, const int new_port); + +int storage_open_readable_binlog(StorageBinLogReader *pReader, \ + get_filename_func filename_func, const void *pArg); + +int storage_reader_init(FDFSStorageBrief *pStorage, StorageBinLogReader *pReader); +void storage_reader_destroy(StorageBinLogReader *pReader); + +int storage_report_storage_status(const char *storage_id, \ + const char *ip_addr, const char status); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/tracker_client_thread.c b/storage/tracker_client_thread.c new file mode 100644 index 0000000..6515c37 --- /dev/null +++ b/storage/tracker_client_thread.c @@ -0,0 +1,2367 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_client_thread.h" +#include "storage_global.h" +#include "storage_sync.h" +#include "storage_func.h" +#include "tracker_client.h" +#include "trunk_mem.h" +#include "trunk_sync.h" +#include "storage_param_getter.h" + +#define TRUNK_FILE_CREATOR_TASK_ID 88 + +static pthread_mutex_t reporter_thread_lock; + +/* save report thread ids */ +static pthread_t *report_tids = NULL; +static int *src_storage_status = NULL; //returned by tracker server +static signed char *my_report_status = NULL; //returned by tracker server + +static int tracker_heart_beat(ConnectionInfo *pTrackerServer, \ + int *pstat_chg_sync_count, bool *bServerPortChanged); +static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \ + bool *bServerPortChanged); +static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer, \ + bool *bServerPortChanged); + +static int tracker_sync_dest_req(ConnectionInfo *pTrackerServer); +static int tracker_sync_dest_query(ConnectionInfo *pTrackerServer); +static int tracker_sync_notify(ConnectionInfo *pTrackerServer); +static int tracker_storage_changelog_req(ConnectionInfo *pTrackerServer); +static int tracker_report_trunk_fid(ConnectionInfo *pTrackerServer); +static int tracker_fetch_trunk_fid(ConnectionInfo *pTrackerServer); +static int tracker_report_trunk_free_space(ConnectionInfo *pTrackerServer); + +static bool tracker_insert_into_sorted_servers( \ + FDFSStorageServer *pInsertedServer); + +int tracker_report_init() +{ + int result; + + memset(g_storage_servers, 0, sizeof(g_storage_servers)); + memset(g_sorted_storages, 0, sizeof(g_sorted_storages)); + + if ((result=init_pthread_lock(&reporter_thread_lock)) != 0) + { + return result; + } + + return 0; +} + +int tracker_report_destroy() +{ + int result; + + if ((result=pthread_mutex_destroy(&reporter_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_destroy fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int kill_tracker_report_threads() +{ + int result; + int kill_res; + + if (report_tids == NULL) + { + return 0; + } + + if ((result=pthread_mutex_lock(&reporter_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + kill_res = kill_work_threads(report_tids, g_tracker_reporter_count); + + if ((result=pthread_mutex_unlock(&reporter_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return kill_res; +} + +static void thracker_report_thread_exit(ConnectionInfo *pTrackerServer) +{ + int result; + int i; + pthread_t tid; + + if ((result=pthread_mutex_lock(&reporter_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + tid = pthread_self(); + for (i=0; iip_addr, pTrackerServer->port); +} + +static int tracker_unlink_mark_files(const char *storage_id) +{ + int result; + + result = storage_unlink_mark_file(storage_id); + result += trunk_unlink_mark_file(storage_id); + + return result; +} + +static int tracker_rename_mark_files(const char *old_ip_addr, \ + const int old_port, const char *new_ip_addr, const int new_port) +{ + int result; + + result = storage_rename_mark_file(old_ip_addr, old_port, \ + new_ip_addr, new_port); + result += trunk_rename_mark_file(old_ip_addr, old_port, \ + new_ip_addr, new_port); + return result; +} + +static void *tracker_report_thread_entrance(void *arg) +{ + ConnectionInfo *pTrackerServer; + char my_server_id[FDFS_STORAGE_ID_MAX_SIZE]; + char tracker_client_ip[IP_ADDRESS_SIZE]; + char szFailPrompt[36]; + bool sync_old_done; + int stat_chg_sync_count; + int sync_time_chg_count; + time_t current_time; + time_t last_df_report_time; + time_t last_sync_report_time; + time_t last_beat_time; + int last_trunk_file_id; + int result; + int previousCode; + int nContinuousFail; + int tracker_index; + int64_t last_trunk_total_free_space; + bool bServerPortChanged; + + bServerPortChanged = (g_last_server_port != 0) && \ + (g_server_port != g_last_server_port); + + pTrackerServer = (ConnectionInfo *)arg; + pTrackerServer->sock = -1; + tracker_index = pTrackerServer - g_tracker_group.servers; + + logDebug("file: "__FILE__", line: %d, " \ + "report thread to tracker server %s:%d started", \ + __LINE__, pTrackerServer->ip_addr, pTrackerServer->port); + + sync_old_done = g_sync_old_done; + while (g_continue_flag && \ + g_tracker_reporter_count < g_tracker_group.server_count) + { + sleep(1); //waiting for all thread started + } + + result = 0; + previousCode = 0; + nContinuousFail = 0; + while (g_continue_flag) + { + if (pTrackerServer->sock >= 0) + { + close(pTrackerServer->sock); + } + pTrackerServer->sock = socket(AF_INET, SOCK_STREAM, 0); + if(pTrackerServer->sock < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s. program exit!", \ + __LINE__, errno, STRERROR(errno)); + g_continue_flag = false; + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(pTrackerServer->sock, g_bind_addr, 0); + } + + tcpsetserveropt(pTrackerServer->sock, g_fdfs_network_timeout); + + if (tcpsetnonblockopt(pTrackerServer->sock) != 0) + { + nContinuousFail++; + sleep(g_heart_beat_interval); + continue; + } + + if ((result=connectserverbyip_nb(pTrackerServer->sock, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, g_fdfs_connect_timeout)) != 0) + { + if (previousCode != result) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail" \ + ", errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + previousCode = result; + } + + nContinuousFail++; + if (g_continue_flag) + { + sleep(g_heart_beat_interval); + continue; + } + else + { + break; + } + } + + getSockIpaddr(pTrackerServer->sock, \ + tracker_client_ip, IP_ADDRESS_SIZE); + + if (nContinuousFail == 0) + { + *szFailPrompt = '\0'; + } + else + { + sprintf(szFailPrompt, ", continuous fail count: %d", \ + nContinuousFail); + } + logInfo("file: "__FILE__", line: %d, " \ + "successfully connect to tracker server %s:%d%s, " \ + "as a tracker client, my ip is %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, szFailPrompt, tracker_client_ip); + + previousCode = 0; + nContinuousFail = 0; + + if (*g_tracker_client_ip == '\0') + { + strcpy(g_tracker_client_ip, tracker_client_ip); + } + else if (strcmp(tracker_client_ip, g_tracker_client_ip) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "as a client of tracker server %s:%d, " \ + "my ip: %s != client ip: %s of other " \ + "tracker client", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + tracker_client_ip, g_tracker_client_ip); + + close(pTrackerServer->sock); + pTrackerServer->sock = -1; + break; + } + + insert_into_local_host_ip(tracker_client_ip); + + /* + //printf("file: "__FILE__", line: %d, " \ + "tracker_client_ip: %s, g_my_server_id_str: %s\n", \ + __LINE__, tracker_client_ip, g_my_server_id_str); + //print_local_host_ip_addrs(); + */ + + if (tracker_report_join(pTrackerServer, tracker_index, \ + sync_old_done) != 0) + { + sleep(g_heart_beat_interval); + continue; + } + + if (g_http_port != g_last_http_port) + { + g_last_http_port = g_http_port; + if ((result=storage_write_to_sync_ini_file()) != 0) + { + } + } + + if (!sync_old_done) + { + if ((result=pthread_mutex_lock(&reporter_thread_lock)) \ + != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + + fdfs_quit(pTrackerServer); + sleep(g_heart_beat_interval); + continue; + } + + if (!g_sync_old_done) + { + if (tracker_sync_dest_req(pTrackerServer) == 0) + { + g_sync_old_done = true; + if (storage_write_to_sync_ini_file() \ + != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "storage_write_to_sync_ini_file"\ + " fail, program exit!", \ + __LINE__); + + g_continue_flag = false; + pthread_mutex_unlock( \ + &reporter_thread_lock); + break; + } + } + else //request failed or need to try again + { + pthread_mutex_unlock( \ + &reporter_thread_lock); + + fdfs_quit(pTrackerServer); + sleep(g_heart_beat_interval); + continue; + } + } + else + { + if (tracker_sync_notify(pTrackerServer) != 0) + { + pthread_mutex_unlock( \ + &reporter_thread_lock); + fdfs_quit(pTrackerServer); + sleep(g_heart_beat_interval); + continue; + } + } + + if ((result=pthread_mutex_unlock(&reporter_thread_lock)) + != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + sync_old_done = true; + } + + src_storage_status[tracker_index] = \ + tracker_sync_notify(pTrackerServer); + if (src_storage_status[tracker_index] != 0) + { + int k; + for (k=0; k= \ + g_heart_beat_interval) + { + if (tracker_heart_beat(pTrackerServer, \ + &stat_chg_sync_count, \ + &bServerPortChanged) != 0) + { + break; + } + + if (g_storage_ip_changed_auto_adjust && \ + tracker_storage_changelog_req( \ + pTrackerServer) != 0) + { + break; + } + + last_beat_time = current_time; + } + + if (sync_time_chg_count != g_sync_change_count && \ + current_time - last_sync_report_time >= \ + g_heart_beat_interval) + { + if (tracker_report_sync_timestamp( \ + pTrackerServer, &bServerPortChanged)!=0) + { + break; + } + + sync_time_chg_count = g_sync_change_count; + last_sync_report_time = current_time; + } + + if (current_time - last_df_report_time >= \ + g_stat_report_interval) + { + if (tracker_report_df_stat(pTrackerServer, \ + &bServerPortChanged) != 0) + { + break; + } + + last_df_report_time = current_time; + } + + if (g_if_trunker_self) + { + if (last_trunk_file_id < g_current_trunk_file_id) + { + if (tracker_report_trunk_fid(pTrackerServer)!=0) + { + break; + } + last_trunk_file_id = g_current_trunk_file_id; + } + + if (last_trunk_total_free_space != g_trunk_total_free_space) + { + if (tracker_report_trunk_free_space(pTrackerServer)!=0) + { + break; + } + last_trunk_total_free_space = g_trunk_total_free_space; + } + } + + sleep(1); + } + + if ((!g_continue_flag) && fdfs_quit(pTrackerServer) != 0) + { + } + + close(pTrackerServer->sock); + pTrackerServer->sock = -1; + if (g_continue_flag) + { + sleep(1); + } + } + + if (nContinuousFail > 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, try count: %d" \ + ", errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, nContinuousFail, \ + result, STRERROR(result)); + } + + thracker_report_thread_exit(pTrackerServer); + + return NULL; +} + +static bool tracker_insert_into_sorted_servers( \ + FDFSStorageServer *pInsertedServer) +{ + FDFSStorageServer **ppServer; + FDFSStorageServer **ppEnd; + int nCompare; + + ppEnd = g_sorted_storages + g_storage_count; + for (ppServer=ppEnd; ppServer > g_sorted_storages; ppServer--) + { + nCompare = strcmp(pInsertedServer->server.id, \ + (*(ppServer-1))->server.id); + if (nCompare > 0) + { + *ppServer = pInsertedServer; + return true; + } + else if (nCompare < 0) + { + *ppServer = *(ppServer-1); + } + else //nCompare == 0 + { + for (; ppServer < ppEnd; ppServer++) //restore + { + *ppServer = *(ppServer+1); + } + return false; + } + } + + *ppServer = pInsertedServer; + return true; +} + +int tracker_sync_diff_servers(ConnectionInfo *pTrackerServer, \ + FDFSStorageBrief *briefServers, const int server_count) +{ + TrackerHeader resp; + int out_len; + int result; + + if (server_count == 0) + { + return 0; + } + + memset(&resp, 0, sizeof(resp)); + resp.cmd = TRACKER_PROTO_CMD_STORAGE_REPLICA_CHG; + + out_len = sizeof(FDFSStorageBrief) * server_count; + long2buff(out_len, resp.pkg_len); + if ((result=tcpsenddata_nb(pTrackerServer->sock, &resp, sizeof(resp), \ + g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "trackert server %s:%d, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result; + } + + if ((result=tcpsenddata_nb(pTrackerServer->sock, \ + briefServers, out_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "trackert server %s:%d, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result; + } + + + if ((result=tcprecvdata_nb(pTrackerServer->sock, &resp, \ + sizeof(resp), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result; + } + + if (memcmp(resp.pkg_len, "\0\0\0\0\0\0\0\0", \ + FDFS_PROTO_PKG_LEN_SIZE) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "expect pkg len 0, but recv pkg len != 0", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port); + return EINVAL; + } + + return resp.status; +} + +int tracker_report_storage_status(ConnectionInfo *pTrackerServer, \ + FDFSStorageBrief *briefServer) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + sizeof(FDFSStorageBrief)]; + TrackerHeader *pHeader; + TrackerHeader resp; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_STATUS; + + long2buff(FDFS_GROUP_NAME_MAX_LEN + sizeof(FDFSStorageBrief), \ + pHeader->pkg_len); + strcpy(out_buff + sizeof(TrackerHeader), g_group_name); + memcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN, \ + briefServer, sizeof(FDFSStorageBrief)); + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + sizeof(FDFSStorageBrief), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "trackert server %s:%d, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result; + } + + if ((result=tcprecvdata_nb(pTrackerServer->sock, &resp, \ + sizeof(resp), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, result, STRERROR(result)); + return result; + } + + if (memcmp(resp.pkg_len, "\0\0\0\0\0\0\0\0", \ + FDFS_PROTO_PKG_LEN_SIZE) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "expect pkg len 0, but recv pkg len != 0", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port); + return EINVAL; + } + + return resp.status; +} + +static int tracker_start_sync_threads(const FDFSStorageBrief *pStorage) +{ + int result; + + if (strcmp(pStorage->id, g_my_server_id_str) == 0) + { + return 0; + } + + result = storage_sync_thread_start(pStorage); + if (result == 0) + { + if (g_if_trunker_self) + { + result = trunk_sync_thread_start(pStorage); + } + } + + return result; +} + +static int tracker_merge_servers(ConnectionInfo *pTrackerServer, \ + FDFSStorageBrief *briefServers, const int server_count) +{ + FDFSStorageBrief *pServer; + FDFSStorageBrief *pEnd; + FDFSStorageServer *pInsertedServer; + FDFSStorageServer **ppFound; + FDFSStorageServer **ppGlobalServer; + FDFSStorageServer **ppGlobalEnd; + FDFSStorageServer targetServer; + FDFSStorageServer *pTargetServer; + FDFSStorageBrief diffServers[FDFS_MAX_SERVERS_EACH_GROUP]; + FDFSStorageBrief *pDiffServer; + int res; + int result; + int nDeletedCount; + + memset(&targetServer, 0, sizeof(targetServer)); + pTargetServer = &targetServer; + + nDeletedCount = 0; + pDiffServer = diffServers; + pEnd = briefServers + server_count; + for (pServer=briefServers; pServerserver.ip_addr, pServer->ip_addr); + } + + /* + //logInfo("ip_addr=%s, local status: %d, " \ + "tracker status: %d", pServer->ip_addr, \ + (*ppFound)->server.status, pServer->status); + */ + if ((*ppFound)->server.status == pServer->status) + { + continue; + } + + if (pServer->status == FDFS_STORAGE_STATUS_OFFLINE) + { + if ((*ppFound)->server.status == \ + FDFS_STORAGE_STATUS_ACTIVE + || (*ppFound)->server.status == \ + FDFS_STORAGE_STATUS_ONLINE) + { + (*ppFound)->server.status = \ + FDFS_STORAGE_STATUS_OFFLINE; + } + else if ((*ppFound)->server.status != \ + FDFS_STORAGE_STATUS_NONE + && (*ppFound)->server.status != \ + FDFS_STORAGE_STATUS_INIT) + { + memcpy(pDiffServer++, \ + &((*ppFound)->server), \ + sizeof(FDFSStorageBrief)); + } + } + else if ((*ppFound)->server.status == \ + FDFS_STORAGE_STATUS_OFFLINE) + { + (*ppFound)->server.status = pServer->status; + } + else if ((*ppFound)->server.status == \ + FDFS_STORAGE_STATUS_NONE) + { + if (pServer->status == \ + FDFS_STORAGE_STATUS_DELETED \ + || pServer->status == \ + FDFS_STORAGE_STATUS_IP_CHANGED) + { //ignore + } + else + { + (*ppFound)->server.status = \ + pServer->status; + if ((result=tracker_start_sync_threads(\ + &((*ppFound)->server))) != 0) + { + return result; + } + } + } + else if (((pServer->status == \ + FDFS_STORAGE_STATUS_WAIT_SYNC) || \ + (pServer->status == \ + FDFS_STORAGE_STATUS_SYNCING)) && \ + ((*ppFound)->server.status > pServer->status)) + { + memcpy(pDiffServer++, &((*ppFound)->server), \ + sizeof(FDFSStorageBrief)); + } + else + { + (*ppFound)->server.status = pServer->status; + } + } + else if (pServer->status == FDFS_STORAGE_STATUS_DELETED + || pServer->status == FDFS_STORAGE_STATUS_IP_CHANGED) + { //ignore + nDeletedCount++; + } + else + { + /* + //logInfo("ip_addr=%s, tracker status: %d", + pServer->ip_addr, pServer->status); + */ + + if ((res=pthread_mutex_lock( \ + &reporter_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, "\ + "call pthread_mutex_lock fail,"\ + " errno: %d, error info: %s", \ + __LINE__, res, STRERROR(res)); + } + + if (g_storage_count < FDFS_MAX_SERVERS_EACH_GROUP) + { + pInsertedServer = g_storage_servers + \ + g_storage_count; + memcpy(&(pInsertedServer->server), \ + pServer, sizeof(FDFSStorageBrief)); + if (tracker_insert_into_sorted_servers( \ + pInsertedServer)) + { + g_storage_count++; + + result = tracker_start_sync_threads( \ + &(pInsertedServer->server)); + } + else + { + result = 0; + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "storage servers of group \"%s\" " \ + "exceeds max: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, g_group_name, \ + FDFS_MAX_SERVERS_EACH_GROUP); + result = ENOSPC; + } + + if ((res=pthread_mutex_unlock( \ + &reporter_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, "\ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, res, STRERROR(res)); + } + + if (result != 0) + { + return result; + } + } + } + + if (g_storage_count + nDeletedCount == server_count) + { + if (pDiffServer - diffServers > 0) + { + return tracker_sync_diff_servers(pTrackerServer, \ + diffServers, pDiffServer - diffServers); + } + + return 0; + } + + ppGlobalServer = g_sorted_storages; + ppGlobalEnd = g_sorted_storages + g_storage_count; + pServer = briefServers; + while (pServer < pEnd && ppGlobalServer < ppGlobalEnd) + { + if ((*ppGlobalServer)->server.status == FDFS_STORAGE_STATUS_NONE) + { + ppGlobalServer++; + continue; + } + + res = strcmp(pServer->id, (*ppGlobalServer)->server.id); + if (res < 0) + { + if (pServer->status != FDFS_STORAGE_STATUS_DELETED + && pServer->status != FDFS_STORAGE_STATUS_IP_CHANGED) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "group \"%s\", " \ + "enter impossible statement branch", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, g_group_name); + } + + pServer++; + } + else if (res == 0) + { + pServer++; + ppGlobalServer++; + } + else + { + memcpy(pDiffServer++, &((*ppGlobalServer)->server), \ + sizeof(FDFSStorageBrief)); + ppGlobalServer++; + } + } + + while (ppGlobalServer < ppGlobalEnd) + { + if ((*ppGlobalServer)->server.status == FDFS_STORAGE_STATUS_NONE) + { + ppGlobalServer++; + continue; + } + + memcpy(pDiffServer++, &((*ppGlobalServer)->server), \ + sizeof(FDFSStorageBrief)); + ppGlobalServer++; + } + + return tracker_sync_diff_servers(pTrackerServer, \ + diffServers, pDiffServer - diffServers); +} + +static int tracker_check_response(ConnectionInfo *pTrackerServer, \ + bool *bServerPortChanged) +{ + int64_t nInPackLen; + TrackerHeader resp; + int server_count; + int result; + char in_buff[1 + (2 + FDFS_MAX_SERVERS_EACH_GROUP) * \ + sizeof(FDFSStorageBrief)]; + FDFSStorageBrief *pBriefServers; + char *pFlags; + + if ((result=tcprecvdata_nb(pTrackerServer->sock, &resp, \ + sizeof(resp), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + //printf("resp status=%d\n", resp.status); + if (resp.status != 0) + { + return resp.status; + } + + nInPackLen = buff2long(resp.pkg_len); + if (nInPackLen == 0) + { + return 0; + } + + if ((nInPackLen <= 0) || ((nInPackLen - 1) % \ + sizeof(FDFSStorageBrief) != 0)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "package size "INT64_PRINTF_FORMAT" is not correct", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, nInPackLen); + return EINVAL; + } + + if (nInPackLen > sizeof(in_buff)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, package size " \ + INT64_PRINTF_FORMAT" is too large, " \ + "exceed max: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + nInPackLen, (int)sizeof(in_buff)); + return EINVAL; + } + + if ((result=tcprecvdata_nb(pTrackerServer->sock, in_buff, \ + nInPackLen, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + pFlags = in_buff; + server_count = (nInPackLen - 1) / sizeof(FDFSStorageBrief); + pBriefServers = (FDFSStorageBrief *)(in_buff + 1); + + if ((*pFlags) & FDFS_CHANGE_FLAG_TRACKER_LEADER) + { + char tracker_leader_ip[IP_ADDRESS_SIZE]; + int tracker_leader_port; + + if (server_count < 1) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, reponse server " \ + "count: %d < 1", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, server_count); + return EINVAL; + } + + memcpy(tracker_leader_ip, pBriefServers->ip_addr, \ + IP_ADDRESS_SIZE - 1); + *(tracker_leader_ip + (IP_ADDRESS_SIZE - 1)) = '\0'; + tracker_leader_port = buff2int(pBriefServers->port); + + if (*tracker_leader_ip == '\0') + { + if (g_tracker_group.leader_index >= 0) + { + ConnectionInfo *pTrackerLeader; + pTrackerLeader = g_tracker_group.servers + \ + g_tracker_group.leader_index; + logWarning("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "my tracker leader is: %s:%d, " \ + "but reponse tracker leader is null", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, pTrackerLeader->ip_addr, \ + pTrackerLeader->port); + + g_tracker_group.leader_index = -1; + } + } + else + { + int leader_index; + + leader_index = fdfs_get_tracker_leader_index( \ + tracker_leader_ip, tracker_leader_port); + if (leader_index < 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "reponse tracker leader: %s:%d" \ + " not exist in local", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, tracker_leader_ip, \ + tracker_leader_port); + } + else + { + logInfo("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "set tracker leader: %s:%d", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port,\ + tracker_leader_ip, tracker_leader_port); + + g_tracker_group.leader_index = leader_index; + } + } + + pBriefServers += 1; + server_count -= 1; + } + + if ((*pFlags) & FDFS_CHANGE_FLAG_TRUNK_SERVER) + { + if (server_count < 1) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, reponse server " \ + "count: %d < 1", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, server_count); + return EINVAL; + } + + if (!g_if_use_trunk_file) + { + logInfo("file: "__FILE__", line: %d, " \ + "reload parameters from tracker server", \ + __LINE__); + storage_get_params_from_tracker(); + } + + if (!g_if_use_trunk_file) + { + logWarning("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "my g_if_use_trunk_file is false, " \ + "can't support trunk server!", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port); + } + else + { + memcpy(g_trunk_server.ip_addr, pBriefServers->ip_addr, \ + IP_ADDRESS_SIZE - 1); + *(g_trunk_server.ip_addr + (IP_ADDRESS_SIZE - 1)) = '\0'; + g_trunk_server.port = buff2int(pBriefServers->port); + if (is_local_host_ip(g_trunk_server.ip_addr) && \ + g_trunk_server.port == g_server_port) + { + if (g_if_trunker_self) + { + logWarning("file: "__FILE__", line: %d, " \ + "I am already the trunk server %s:%d, " \ + "may be the tracker server restart", \ + __LINE__, g_trunk_server.ip_addr, \ + g_trunk_server.port); + } + else + { + logInfo("file: "__FILE__", line: %d, " \ + "I am the the trunk server %s:%d", __LINE__, \ + g_trunk_server.ip_addr, g_trunk_server.port); + + tracker_fetch_trunk_fid(pTrackerServer); + g_if_trunker_self = true; + + if ((result=storage_trunk_init()) != 0) + { + return result; + } + + if (g_trunk_create_file_advance && \ + g_trunk_create_file_interval > 0) + { + ScheduleArray scheduleArray; + ScheduleEntry entries[1]; + + entries[0].id = TRUNK_FILE_CREATOR_TASK_ID; + entries[0].time_base = g_trunk_create_file_time_base; + entries[0].interval = g_trunk_create_file_interval; + entries[0].task_func = trunk_create_trunk_file_advance; + entries[0].func_args = NULL; + + scheduleArray.count = 1; + scheduleArray.entries = entries; + sched_add_entries(&scheduleArray); + } + + trunk_sync_thread_start_all(); + } + } + else + { + logInfo("file: "__FILE__", line: %d, " \ + "the trunk server is %s:%d", __LINE__, \ + g_trunk_server.ip_addr, g_trunk_server.port); + + if (g_if_trunker_self) + { + int saved_trunk_sync_thread_count; + + logWarning("file: "__FILE__", line: %d, " \ + "I am the old trunk server, " \ + "the new trunk server is %s:%d", \ + __LINE__, g_trunk_server.ip_addr, \ + g_trunk_server.port); + + tracker_report_trunk_fid(pTrackerServer); + g_if_trunker_self = false; + + saved_trunk_sync_thread_count = \ + g_trunk_sync_thread_count; + if (saved_trunk_sync_thread_count > 0) + { + logInfo("file: "__FILE__", line: %d, "\ + "waiting %d trunk sync " \ + "threads exit ...", __LINE__, \ + saved_trunk_sync_thread_count); + } + + while (g_trunk_sync_thread_count > 0) + { + usleep(50000); + } + + if (saved_trunk_sync_thread_count > 0) + { + logInfo("file: "__FILE__", line: %d, " \ + "%d trunk sync threads exited",\ + __LINE__, \ + saved_trunk_sync_thread_count); + } + + storage_trunk_destroy_ex(true); + if (g_trunk_create_file_advance && \ + g_trunk_create_file_interval > 0) + { + sched_del_entry(TRUNK_FILE_CREATOR_TASK_ID); + } + } + } + } + + pBriefServers += 1; + server_count -= 1; + } + + if (!((*pFlags) & FDFS_CHANGE_FLAG_GROUP_SERVER)) + { + return 0; + } + + /* + //printf("resp server count=%d\n", server_count); + { + int i; + for (i=0; iid, g_my_server_id_str) == 0) + { + continue; + } + + tracker_rename_mark_files(pStorage->ip_addr, \ + g_last_server_port, pStorage->ip_addr, \ + g_server_port); + } + } + + if (g_server_port != g_last_server_port) + { + g_last_server_port = g_server_port; + if ((result=storage_write_to_sync_ini_file()) != 0) + { + return result; + } + } + } + + return tracker_merge_servers(pTrackerServer, \ + pBriefServers, server_count); +} + +int tracker_sync_src_req(ConnectionInfo *pTrackerServer, \ + StorageBinLogReader *pReader) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_STORAGE_ID_MAX_SIZE]; + char sync_src_id[FDFS_STORAGE_ID_MAX_SIZE]; + TrackerHeader *pHeader; + TrackerStorageSyncReqBody syncReqbody; + char *pBuff; + int64_t in_bytes; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + long2buff(FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE, \ + pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_SRC_REQ; + strcpy(out_buff + sizeof(TrackerHeader), g_group_name); + strcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN, \ + pReader->storage_id); + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = (char *)&syncReqbody; + if ((result=fdfs_recv_response(pTrackerServer, \ + &pBuff, sizeof(syncReqbody), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes == 0) + { + pReader->need_sync_old = false; + pReader->until_timestamp = 0; + + return 0; + } + + if (in_bytes != sizeof(syncReqbody)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "recv body length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect body length: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes, \ + (int)sizeof(syncReqbody)); + return EINVAL; + } + + memcpy(sync_src_id, syncReqbody.src_id, FDFS_STORAGE_ID_MAX_SIZE); + sync_src_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0'; + + pReader->need_sync_old = storage_id_is_myself(sync_src_id); + pReader->until_timestamp = (time_t)buff2long( \ + syncReqbody.until_timestamp); + + return 0; +} + +static int tracker_sync_dest_req(ConnectionInfo *pTrackerServer) +{ + TrackerHeader header; + TrackerStorageSyncReqBody syncReqbody; + char *pBuff; + int64_t in_bytes; + int result; + + memset(&header, 0, sizeof(header)); + header.cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ; + if ((result=tcpsenddata_nb(pTrackerServer->sock, &header, \ + sizeof(header), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = (char *)&syncReqbody; + if ((result=fdfs_recv_response(pTrackerServer, \ + &pBuff, sizeof(syncReqbody), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes == 0) + { + return result; + } + + if (in_bytes != sizeof(syncReqbody)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "recv body length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect body length: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes, \ + (int)sizeof(syncReqbody)); + return EINVAL; + } + + memcpy(g_sync_src_id, syncReqbody.src_id, FDFS_STORAGE_ID_MAX_SIZE); + g_sync_src_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0'; + + g_sync_until_timestamp = (time_t)buff2long(syncReqbody.until_timestamp); + + return 0; +} + +static int tracker_sync_dest_query(ConnectionInfo *pTrackerServer) +{ + TrackerHeader header; + TrackerStorageSyncReqBody syncReqbody; + char *pBuff; + int64_t in_bytes; + int result; + + memset(&header, 0, sizeof(header)); + header.cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_QUERY; + if ((result=tcpsenddata_nb(pTrackerServer->sock, &header, \ + sizeof(header), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = (char *)&syncReqbody; + if ((result=fdfs_recv_response(pTrackerServer, \ + &pBuff, sizeof(syncReqbody), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes == 0) + { + *g_sync_src_id = '\0'; + g_sync_until_timestamp = 0; + return result; + } + + if (in_bytes != sizeof(syncReqbody)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "recv body length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect body length: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes, \ + (int)sizeof(syncReqbody)); + return EINVAL; + } + + memcpy(g_sync_src_id, syncReqbody.src_id, FDFS_STORAGE_ID_MAX_SIZE); + g_sync_src_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0'; + + g_sync_until_timestamp = (time_t)buff2long(syncReqbody.until_timestamp); + return 0; +} + +static int tracker_report_trunk_fid(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader)+sizeof(int)]; + TrackerHeader *pHeader; + int64_t in_bytes; + int result; + + pHeader = (TrackerHeader *)out_buff; + + memset(out_buff, 0, sizeof(out_buff)); + long2buff((int)sizeof(int), pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID; + int2buff(g_current_trunk_file_id, out_buff + sizeof(TrackerHeader)); + + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=fdfs_recv_header(pTrackerServer, &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv body length: " \ + INT64_PRINTF_FORMAT" != 0", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +static int tracker_report_trunk_free_space(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader) + 8]; + TrackerHeader *pHeader; + int64_t in_bytes; + int result; + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + long2buff(8, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FREE; + long2buff(g_trunk_total_free_space / FDFS_ONE_MB, \ + out_buff + sizeof(TrackerHeader)); + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=fdfs_recv_header(pTrackerServer, &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv body length: " \ + INT64_PRINTF_FORMAT" != 0", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +static int tracker_fetch_trunk_fid(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader)]; + char in_buff[4]; + TrackerHeader *pHeader; + char *pInBuff; + int64_t in_bytes; + int trunk_fid; + int result; + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_FETCH_TRUNK_FID; + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + pInBuff = in_buff; + if ((result=fdfs_recv_response(pTrackerServer, \ + &pInBuff, sizeof(in_buff), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != sizeof(in_buff)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv body length: " \ + INT64_PRINTF_FORMAT" != %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes, (int)sizeof(in_buff)); + return EINVAL; + } + + trunk_fid = buff2int(in_buff); + if (trunk_fid < 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, " \ + "trunk file id: %d is invalid!", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, trunk_fid); + return EINVAL; + } + + if (g_current_trunk_file_id < trunk_fid) + { + logInfo("file: "__FILE__", line: %d, " \ + "old trunk file id: %d, " \ + "change to new trunk file id: %d", \ + __LINE__, g_current_trunk_file_id, trunk_fid); + + g_current_trunk_file_id = trunk_fid; + storage_write_to_sync_ini_file(); + } + + return 0; +} + +static int tracker_sync_notify(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader)+sizeof(TrackerStorageSyncReqBody)]; + TrackerHeader *pHeader; + TrackerStorageSyncReqBody *pReqBody; + int64_t in_bytes; + int result; + + pHeader = (TrackerHeader *)out_buff; + pReqBody = (TrackerStorageSyncReqBody*)(out_buff+sizeof(TrackerHeader)); + + memset(out_buff, 0, sizeof(out_buff)); + long2buff((int)sizeof(TrackerStorageSyncReqBody), pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY; + strcpy(pReqBody->src_id, g_sync_src_id); + long2buff(g_sync_until_timestamp, pReqBody->until_timestamp); + + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=fdfs_recv_header(pTrackerServer, &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv body length: " \ + INT64_PRINTF_FORMAT" != 0", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +int tracker_report_join(ConnectionInfo *pTrackerServer, \ + const int tracker_index, const bool sync_old_done) +{ + char out_buff[sizeof(TrackerHeader) + sizeof(TrackerStorageJoinBody) + \ + FDFS_MAX_TRACKERS * FDFS_PROTO_IP_PORT_SIZE]; + TrackerHeader *pHeader; + TrackerStorageJoinBody *pReqBody; + TrackerStorageJoinBodyResp respBody; + char *pInBuff; + char *p; + ConnectionInfo *pServer; + ConnectionInfo *pServerEnd; + FDFSStorageServer *pTargetServer; + FDFSStorageServer **ppFound; + FDFSStorageServer targetServer; + int out_len; + //int tracker_count; + int result; + int i; + int64_t in_bytes; + + pHeader = (TrackerHeader *)out_buff; + pReqBody = (TrackerStorageJoinBody *)(out_buff+sizeof(TrackerHeader)); + + memset(out_buff, 0, sizeof(out_buff)); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_JOIN; + strcpy(pReqBody->group_name, g_group_name); + strcpy(pReqBody->domain_name, g_http_domain); + snprintf(pReqBody->version, sizeof(pReqBody->version), "%d.%02d", \ + g_fdfs_version.major, g_fdfs_version.minor); + long2buff(g_server_port, pReqBody->storage_port); + long2buff(g_http_port, pReqBody->storage_http_port); + long2buff(g_fdfs_store_paths.count, pReqBody->store_path_count); + long2buff(g_subdir_count_per_path, pReqBody->subdir_count_per_path); + long2buff(g_upload_priority, pReqBody->upload_priority); + long2buff(g_storage_join_time, pReqBody->join_time); + long2buff(g_up_time, pReqBody->up_time); + pReqBody->init_flag = sync_old_done ? 0 : 1; + + memset(&targetServer, 0, sizeof(targetServer)); + pTargetServer = &targetServer; + + strcpy(targetServer.server.id, g_my_server_id_str); + ppFound = (FDFSStorageServer **)bsearch(&pTargetServer, \ + g_sorted_storages, g_storage_count, \ + sizeof(FDFSStorageServer *), storage_cmp_by_server_id); + if (ppFound != NULL) + { + pReqBody->status = (*ppFound)->server.status; + } + else + { + if (g_tracker_group.server_count > 1) + { + for (i=0; istatus = FDFS_STORAGE_STATUS_INIT; + } + else + { + pReqBody->status = -1; + } + } + else + { + pReqBody->status = FDFS_STORAGE_STATUS_INIT; + } + } + + //tracker_count = 0; + p = out_buff + sizeof(TrackerHeader) + sizeof(TrackerStorageJoinBody); + pServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + for (pServer=g_tracker_group.servers; pServerip_addr, pTrackerServer->ip_addr) == 0 && \ + pServer->port == pTrackerServer->port) + { + continue; + } + tracker_count++; + */ + + sprintf(p, "%s:%d", pServer->ip_addr, pServer->port); + p += FDFS_PROTO_IP_PORT_SIZE; + } + + out_len = p - out_buff; + long2buff(g_tracker_group.server_count, pReqBody->tracker_count); + long2buff(out_len - (int)sizeof(TrackerHeader), pHeader->pkg_len); + + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + out_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + pInBuff = (char *)&respBody; + result = fdfs_recv_response(pTrackerServer, \ + &pInBuff, sizeof(respBody), &in_bytes); + my_report_status[tracker_index] = result; + if (result != 0) + { + return result; + } + + if (in_bytes != sizeof(respBody)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data fail, " \ + "expect %d bytes, but recv " \ + INT64_PRINTF_FORMAT" bytes", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + (int)sizeof(respBody), in_bytes); + my_report_status[tracker_index] = EINVAL; + return EINVAL; + } + + if (*(respBody.src_id) == '\0' && *g_sync_src_id != '\0') + { + return tracker_sync_notify(pTrackerServer); + } + else + { + return 0; + } +} + +static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer, \ + bool *bServerPortChanged) +{ + char out_buff[sizeof(TrackerHeader) + (FDFS_STORAGE_ID_MAX_SIZE + 4) * \ + FDFS_MAX_SERVERS_EACH_GROUP]; + char *p; + TrackerHeader *pHeader; + FDFSStorageServer *pServer; + FDFSStorageServer *pEnd; + int result; + int body_len; + + if (g_storage_count == 0) + { + return 0; + } + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + + body_len = (FDFS_STORAGE_ID_MAX_SIZE + 4) * g_storage_count; + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT; + long2buff(body_len, pHeader->pkg_len); + + pEnd = g_storage_servers + g_storage_count; + for (pServer=g_storage_servers; pServerserver.id, FDFS_STORAGE_ID_MAX_SIZE); + p += FDFS_STORAGE_ID_MAX_SIZE; + int2buff(pServer->last_sync_src_timestamp, p); + p += 4; + } + + if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(TrackerHeader) + body_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + return tracker_check_response(pTrackerServer, bServerPortChanged); +} + +static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \ + bool *bServerPortChanged) +{ + char out_buff[sizeof(TrackerHeader) + \ + sizeof(TrackerStatReportReqBody) * 16]; + char *pBuff; + TrackerHeader *pHeader; + TrackerStatReportReqBody *pStatBuff; + struct statvfs sbuf; + int body_len; + int total_len; + int store_path_index; + int i; + int result; + + body_len = (int)sizeof(TrackerStatReportReqBody) * g_fdfs_store_paths.count; + total_len = (int)sizeof(TrackerHeader) + body_len; + if (total_len <= sizeof(out_buff)) + { + pBuff = out_buff; + } + else + { + pBuff = (char *)malloc(total_len); + if (pBuff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, total_len, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + } + + pHeader = (TrackerHeader *)pBuff; + pStatBuff = (TrackerStatReportReqBody*) \ + (pBuff + sizeof(TrackerHeader)); + long2buff(body_len, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE; + pHeader->status = 0; + + for (i=0; isz_total_mb); + long2buff(g_path_space_list[i].free_mb, pStatBuff->sz_free_mb); + + pStatBuff++; + } + + if (g_store_path_mode == FDFS_STORE_PATH_LOAD_BALANCE) + { + int max_free_mb; + + /* find the max free space path */ + max_free_mb = 0; + store_path_index = -1; + for (i=0; i \ + g_avg_storage_reserved_mb \ + && g_path_space_list[i].free_mb > max_free_mb) + { + store_path_index = i; + max_free_mb = g_path_space_list[i].free_mb; + } + } + if (g_store_path_index != store_path_index) + { + g_store_path_index = store_path_index; + } + } + + result = tcpsenddata_nb(pTrackerServer->sock, pBuff, \ + total_len, g_fdfs_network_timeout); + if (pBuff != out_buff) + { + free(pBuff); + } + if(result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + return tracker_check_response(pTrackerServer, bServerPortChanged); +} + +static int tracker_heart_beat(ConnectionInfo *pTrackerServer, \ + int *pstat_chg_sync_count, bool *bServerPortChanged) +{ + char out_buff[sizeof(TrackerHeader) + sizeof(FDFSStorageStatBuff)]; + TrackerHeader *pHeader; + FDFSStorageStatBuff *pStatBuff; + int body_len; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + if (*pstat_chg_sync_count != g_stat_change_count) + { + pStatBuff = (FDFSStorageStatBuff *)( \ + out_buff + sizeof(TrackerHeader)); + long2buff(g_storage_stat.total_upload_count, \ + pStatBuff->sz_total_upload_count); + long2buff(g_storage_stat.success_upload_count, \ + pStatBuff->sz_success_upload_count); + long2buff(g_storage_stat.total_append_count, \ + pStatBuff->sz_total_append_count); + long2buff(g_storage_stat.success_append_count, \ + pStatBuff->sz_success_append_count); + long2buff(g_storage_stat.total_modify_count, \ + pStatBuff->sz_total_modify_count); + long2buff(g_storage_stat.success_modify_count, \ + pStatBuff->sz_success_modify_count); + long2buff(g_storage_stat.total_truncate_count, \ + pStatBuff->sz_total_truncate_count); + long2buff(g_storage_stat.success_truncate_count, \ + pStatBuff->sz_success_truncate_count); + long2buff(g_storage_stat.total_download_count, \ + pStatBuff->sz_total_download_count); + long2buff(g_storage_stat.success_download_count, \ + pStatBuff->sz_success_download_count); + long2buff(g_storage_stat.total_set_meta_count, \ + pStatBuff->sz_total_set_meta_count); + long2buff(g_storage_stat.success_set_meta_count, \ + pStatBuff->sz_success_set_meta_count); + long2buff(g_storage_stat.total_delete_count, \ + pStatBuff->sz_total_delete_count); + long2buff(g_storage_stat.success_delete_count, \ + pStatBuff->sz_success_delete_count); + long2buff(g_storage_stat.total_get_meta_count, \ + pStatBuff->sz_total_get_meta_count); + long2buff(g_storage_stat.success_get_meta_count, \ + pStatBuff->sz_success_get_meta_count); + long2buff(g_storage_stat.total_create_link_count, \ + pStatBuff->sz_total_create_link_count); + long2buff(g_storage_stat.success_create_link_count, \ + pStatBuff->sz_success_create_link_count); + long2buff(g_storage_stat.total_delete_link_count, \ + pStatBuff->sz_total_delete_link_count); + long2buff(g_storage_stat.success_delete_link_count, \ + pStatBuff->sz_success_delete_link_count); + long2buff(g_storage_stat.total_upload_bytes, \ + pStatBuff->sz_total_upload_bytes); + long2buff(g_storage_stat.success_upload_bytes, \ + pStatBuff->sz_success_upload_bytes); + long2buff(g_storage_stat.total_append_bytes, \ + pStatBuff->sz_total_append_bytes); + long2buff(g_storage_stat.success_append_bytes, \ + pStatBuff->sz_success_append_bytes); + long2buff(g_storage_stat.total_modify_bytes, \ + pStatBuff->sz_total_modify_bytes); + long2buff(g_storage_stat.success_modify_bytes, \ + pStatBuff->sz_success_modify_bytes); + long2buff(g_storage_stat.total_download_bytes, \ + pStatBuff->sz_total_download_bytes); + long2buff(g_storage_stat.success_download_bytes, \ + pStatBuff->sz_success_download_bytes); + long2buff(g_storage_stat.total_sync_in_bytes, \ + pStatBuff->sz_total_sync_in_bytes); + long2buff(g_storage_stat.success_sync_in_bytes, \ + pStatBuff->sz_success_sync_in_bytes); + long2buff(g_storage_stat.total_sync_out_bytes, \ + pStatBuff->sz_total_sync_out_bytes); + long2buff(g_storage_stat.success_sync_out_bytes, \ + pStatBuff->sz_success_sync_out_bytes); + long2buff(g_storage_stat.total_file_open_count, \ + pStatBuff->sz_total_file_open_count); + long2buff(g_storage_stat.success_file_open_count, \ + pStatBuff->sz_success_file_open_count); + long2buff(g_storage_stat.total_file_read_count, \ + pStatBuff->sz_total_file_read_count); + long2buff(g_storage_stat.success_file_read_count, \ + pStatBuff->sz_success_file_read_count); + long2buff(g_storage_stat.total_file_write_count, \ + pStatBuff->sz_total_file_write_count); + long2buff(g_storage_stat.success_file_write_count, \ + pStatBuff->sz_success_file_write_count); + long2buff(g_storage_stat.last_source_update, \ + pStatBuff->sz_last_source_update); + long2buff(g_storage_stat.last_sync_update, \ + pStatBuff->sz_last_sync_update); + + *pstat_chg_sync_count = g_stat_change_count; + body_len = sizeof(FDFSStorageStatBuff); + } + else + { + body_len = 0; + } + + long2buff(body_len, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_BEAT; + + if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(TrackerHeader) + body_len, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + return tracker_check_response(pTrackerServer, bServerPortChanged); +} + +static int tracker_storage_changelog_req(ConnectionInfo *pTrackerServer) +{ + char out_buff[sizeof(TrackerHeader)]; + TrackerHeader *pHeader; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + + long2buff(0, pHeader->pkg_len); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ; + + if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(TrackerHeader), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + return tracker_deal_changelog_response(pTrackerServer); +} + +int tracker_deal_changelog_response(ConnectionInfo *pTrackerServer) +{ +#define FDFS_CHANGELOG_FIELDS 5 + int64_t nInPackLen; + char *pInBuff; + char *pBuffEnd; + char *pLineStart; + char *pLineEnd; + char *cols[FDFS_CHANGELOG_FIELDS + 1]; + char *pGroupName; + char *pOldStorageId; + char *pNewStorageId; + char szLine[256]; + int server_status; + int col_count; + int result; + + pInBuff = NULL; + result = fdfs_recv_response(pTrackerServer, \ + &pInBuff, 0, &nInPackLen); + if (result != 0) + { + return result; + } + + if (nInPackLen == 0) + { + return result; + } + + *(pInBuff + nInPackLen) = '\0'; + + pLineStart = pInBuff; + pBuffEnd = pInBuff + nInPackLen; + while (pLineStart < pBuffEnd) + { + if (*pLineStart == '\0') //skip empty line + { + pLineStart++; + continue; + } + + pLineEnd = strchr(pLineStart, '\n'); + if (pLineEnd != NULL) + { + *pLineEnd = '\0'; + } + + snprintf(szLine, sizeof(szLine), "%s", pLineStart); + col_count = splitEx(szLine, ' ', cols, \ + FDFS_CHANGELOG_FIELDS + 1); + + do + { + if (col_count != FDFS_CHANGELOG_FIELDS) + { + logError("file: "__FILE__", line: %d, " \ + "changelog line field count: %d != %d,"\ + "line content=%s", __LINE__, col_count,\ + FDFS_CHANGELOG_FIELDS, pLineStart); + break; + } + + pGroupName = cols[1]; + if (strcmp(pGroupName, g_group_name) != 0) + { //ignore other group's changelog + break; + } + + pOldStorageId = cols[2]; + server_status = atoi(cols[3]); + pNewStorageId = cols[4]; + + if (server_status == FDFS_STORAGE_STATUS_DELETED) + { + tracker_unlink_mark_files(pOldStorageId); + + if (strcmp(g_sync_src_id, pOldStorageId) == 0) + { + *g_sync_src_id = '\0'; + storage_write_to_sync_ini_file(); + } + } + else if (server_status == FDFS_STORAGE_STATUS_IP_CHANGED) + { + if (!g_use_storage_id) + { + tracker_rename_mark_files(pOldStorageId, \ + g_server_port, pNewStorageId, g_server_port); + if (strcmp(g_sync_src_id, pOldStorageId) == 0) + { + snprintf(g_sync_src_id, \ + sizeof(g_sync_src_id), \ + "%s", pNewStorageId); + storage_write_to_sync_ini_file(); + } + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "invalid status: %d in changelog, " \ + "line content=%s", __LINE__, \ + server_status, pLineStart); + } + } while (0); + + if (pLineEnd == NULL) + { + break; + } + + pLineStart = pLineEnd + 1; + } + + free(pInBuff); + + return 0; +} + +int tracker_report_thread_start() +{ + ConnectionInfo *pTrackerServer; + ConnectionInfo *pServerEnd; + pthread_attr_t pattr; + pthread_t tid; + int result; + + if ((result=init_pthread_attr(&pattr, g_thread_stack_size)) != 0) + { + return result; + } + + report_tids = (pthread_t *)malloc(sizeof(pthread_t) * \ + g_tracker_group.server_count); + if (report_tids == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(pthread_t) * \ + g_tracker_group.server_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(report_tids, 0, sizeof(pthread_t)*g_tracker_group.server_count); + + src_storage_status = (int *)malloc(sizeof(int) * \ + g_tracker_group.server_count); + if (src_storage_status == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(int) * g_tracker_group.server_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(src_storage_status,-1,sizeof(int)*g_tracker_group.server_count); + + my_report_status = (signed char *)malloc(sizeof(signed char) * \ + g_tracker_group.server_count); + if (my_report_status == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)sizeof(signed char) * g_tracker_group.server_count, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memset(my_report_status, -1, sizeof(char)*g_tracker_group.server_count); + + g_tracker_reporter_count = 0; + pServerEnd = g_tracker_group.servers + g_tracker_group.server_count; + for (pTrackerServer=g_tracker_group.servers; pTrackerServer +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "trunk_client.h" + +static int trunk_client_trunk_do_alloc_space(ConnectionInfo *pTrunkServer, \ + const int file_size, FDFSTrunkFullInfo *pTrunkInfo) +{ + TrackerHeader *pHeader; + char *p; + int result; + char out_buff[sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + 5]; + FDFSTrunkInfoBuff trunkBuff; + int64_t in_bytes; + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + p = out_buff + sizeof(TrackerHeader); + snprintf(p, sizeof(out_buff) - sizeof(TrackerHeader), \ + "%s", g_group_name); + p += FDFS_GROUP_NAME_MAX_LEN; + int2buff(file_size, p); + p += 4; + *p++ = pTrunkInfo->path.store_path_index; + long2buff(FDFS_GROUP_NAME_MAX_LEN + 5, pHeader->pkg_len); + pHeader->cmd = STORAGE_PROTO_CMD_TRUNK_ALLOC_SPACE; + + if ((result=tcpsenddata_nb(pTrunkServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrunkServer->ip_addr, pTrunkServer->port, \ + result, STRERROR(result)); + + return result; + } + + p = (char *)&trunkBuff; + if ((result=fdfs_recv_response(pTrunkServer, \ + &p, sizeof(FDFSTrunkInfoBuff), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != sizeof(FDFSTrunkInfoBuff)) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d, recv body length: %d invalid, " \ + "expect body length: %d", __LINE__, \ + pTrunkServer->ip_addr, pTrunkServer->port, \ + (int)in_bytes, (int)sizeof(FDFSTrunkInfoBuff)); + return EINVAL; + } + + pTrunkInfo->path.store_path_index = trunkBuff.store_path_index; + pTrunkInfo->path.sub_path_high = trunkBuff.sub_path_high; + pTrunkInfo->path.sub_path_low = trunkBuff.sub_path_low; + pTrunkInfo->file.id = buff2int(trunkBuff.id); + pTrunkInfo->file.offset = buff2int(trunkBuff.offset); + pTrunkInfo->file.size = buff2int(trunkBuff.size); + pTrunkInfo->status = FDFS_TRUNK_STATUS_HOLD; + + return 0; +} + +int trunk_client_trunk_alloc_space(const int file_size, \ + FDFSTrunkFullInfo *pTrunkInfo) +{ + int result; + ConnectionInfo trunk_server; + ConnectionInfo *pTrunkServer; + + if (g_if_trunker_self) + { + return trunk_alloc_space(file_size, pTrunkInfo); + } + + if (*(g_trunk_server.ip_addr) == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "no trunk server", __LINE__); + return EAGAIN; + } + + memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo)); + if ((pTrunkServer=tracker_connect_server(&trunk_server, &result)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "can't alloc trunk space because connect to trunk " \ + "server %s:%d fail, errno: %d", __LINE__, \ + trunk_server.ip_addr, trunk_server.port, result); + return result; + } + + result = trunk_client_trunk_do_alloc_space(pTrunkServer, \ + file_size, pTrunkInfo); + + tracker_disconnect_server_ex(pTrunkServer, result != 0); + return result; +} + +#define trunk_client_trunk_do_alloc_confirm(pTrunkServer, pTrunkInfo, status) \ + trunk_client_trunk_confirm_or_free(pTrunkServer, pTrunkInfo, \ + STORAGE_PROTO_CMD_TRUNK_ALLOC_CONFIRM, status) + +#define trunk_client_trunk_do_free_space(pTrunkServer, pTrunkInfo) \ + trunk_client_trunk_confirm_or_free(pTrunkServer, pTrunkInfo, \ + STORAGE_PROTO_CMD_TRUNK_FREE_SPACE, 0) + +static int trunk_client_trunk_confirm_or_free(ConnectionInfo *pTrunkServer,\ + const FDFSTrunkFullInfo *pTrunkInfo, const int cmd, \ + const int status) +{ + TrackerHeader *pHeader; + FDFSTrunkInfoBuff *pTrunkBuff; + int64_t in_bytes; + int result; + char out_buff[sizeof(TrackerHeader) \ + + STORAGE_TRUNK_ALLOC_CONFIRM_REQ_BODY_LEN]; + + pHeader = (TrackerHeader *)out_buff; + pTrunkBuff = (FDFSTrunkInfoBuff *)(out_buff + sizeof(TrackerHeader) \ + + FDFS_GROUP_NAME_MAX_LEN); + memset(out_buff, 0, sizeof(out_buff)); + snprintf(out_buff + sizeof(TrackerHeader), sizeof(out_buff) - \ + sizeof(TrackerHeader), "%s", g_group_name); + long2buff(STORAGE_TRUNK_ALLOC_CONFIRM_REQ_BODY_LEN, pHeader->pkg_len); + pHeader->cmd = cmd; + pHeader->status = status; + + pTrunkBuff->store_path_index = pTrunkInfo->path.store_path_index; + pTrunkBuff->sub_path_high = pTrunkInfo->path.sub_path_high; + pTrunkBuff->sub_path_low = pTrunkInfo->path.sub_path_low; + int2buff(pTrunkInfo->file.id, pTrunkBuff->id); + int2buff(pTrunkInfo->file.offset, pTrunkBuff->offset); + int2buff(pTrunkInfo->file.size, pTrunkBuff->size); + + if ((result=tcpsenddata_nb(pTrunkServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrunkServer->ip_addr, pTrunkServer->port, \ + result, STRERROR(result)); + + return result; + } + + if ((result=fdfs_recv_header(pTrunkServer, &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "should == 0", __LINE__, pTrunkServer->ip_addr, \ + pTrunkServer->port, in_bytes); + return EINVAL; + } + + return 0; +} + +int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \ + const int status) +{ + int result; + ConnectionInfo trunk_server; + ConnectionInfo *pTrunkServer; + + if (g_if_trunker_self) + { + return trunk_alloc_confirm(pTrunkInfo, status); + } + + if (*(g_trunk_server.ip_addr) == '\0') + { + return EAGAIN; + } + + memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo)); + if ((pTrunkServer=tracker_connect_server(&trunk_server, &result)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "trunk alloc confirm fail because connect to trunk " \ + "server %s:%d fail, errno: %d", __LINE__, \ + trunk_server.ip_addr, trunk_server.port, result); + return result; + } + + result = trunk_client_trunk_do_alloc_confirm(pTrunkServer, \ + pTrunkInfo, status); + + tracker_disconnect_server_ex(pTrunkServer, result != 0); + return result; +} + +int trunk_client_trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo) +{ + int result; + ConnectionInfo trunk_server; + ConnectionInfo *pTrunkServer; + + if (g_if_trunker_self) + { + return trunk_free_space(pTrunkInfo, true); + } + + if (*(g_trunk_server.ip_addr) == '\0') + { + return EAGAIN; + } + + memcpy(&trunk_server, &g_trunk_server, sizeof(ConnectionInfo)); + if ((pTrunkServer=tracker_connect_server(&trunk_server, &result)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "free trunk space fail because connect to trunk " \ + "server %s:%d fail, errno: %d", __LINE__, \ + trunk_server.ip_addr, trunk_server.port, result); + return result; + } + + result = trunk_client_trunk_do_free_space(pTrunkServer, pTrunkInfo); + tracker_disconnect_server_ex(pTrunkServer, result != 0); + return result; +} + diff --git a/storage/trunk_mgr/trunk_client.h b/storage/trunk_mgr/trunk_client.h new file mode 100644 index 0000000..93413d0 --- /dev/null +++ b/storage/trunk_mgr/trunk_client.h @@ -0,0 +1,35 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_client.h + +#ifndef _TRUNK_CLIENT_H_ +#define _TRUNK_CLIENT_H_ + +#include "common_define.h" +#include "tracker_types.h" +#include "trunk_mem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int trunk_client_trunk_alloc_space(const int file_size, \ + FDFSTrunkFullInfo *pTrunkInfo); + +int trunk_client_trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, \ + const int status); + +int trunk_client_trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/trunk_mgr/trunk_free_block_checker.c b/storage/trunk_mgr/trunk_free_block_checker.c new file mode 100644 index 0000000..03ba8b4 --- /dev/null +++ b/storage/trunk_mgr/trunk_free_block_checker.c @@ -0,0 +1,545 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_free_block_checker.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "shared_func.h" +#include "avl_tree.h" +#include "tracker_types.h" +#include "storage_global.h" +#include "trunk_free_block_checker.h" + +#define TRUNK_FREE_BLOCK_ARRAY_INIT_SIZE 32 + +static AVLTreeInfo tree_info_by_id = {NULL, NULL, NULL}; //for unique block nodes + +static int storage_trunk_node_compare_entry(void *p1, void *p2) +{ + return memcmp(&(((FDFSTrunksById *)p1)->trunk_file_id), \ + &(((FDFSTrunksById *)p2)->trunk_file_id), \ + sizeof(FDFSTrunkFileIdentifier)); +} + +int trunk_free_block_checker_init() +{ + int result; + if ((result=avl_tree_init(&tree_info_by_id, free, \ + storage_trunk_node_compare_entry)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "avl_tree_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +void trunk_free_block_checker_destroy() +{ + avl_tree_destroy(&tree_info_by_id); +} + +int trunk_free_block_tree_node_count() +{ + return avl_tree_count(&tree_info_by_id); +} + +static int block_tree_count_walk_callback(void *data, void *args) +{ + int *pcount; + pcount = (int *)args; + + *pcount += ((FDFSTrunksById *)data)->block_array.count; + return 0; +} + +int trunk_free_block_total_count() +{ + int count; + count = 0; + avl_tree_walk(&tree_info_by_id, block_tree_count_walk_callback, &count); + return count; +} + +#define FILL_FILE_IDENTIFIER(target, pTrunkInfo) \ + memset(&target, 0, sizeof(target)); \ + memcpy(&(target.path), &(pTrunkInfo->path), sizeof(FDFSTrunkPathInfo));\ + target.id = pTrunkInfo->file.id; + +int trunk_free_block_check_duplicate(FDFSTrunkFullInfo *pTrunkInfo) +{ + FDFSTrunkFileIdentifier target; + FDFSTrunksById *pFound; + FDFSTrunkFullInfo **blocks; + int end_offset; + int left; + int right; + int mid; + int result; + + /* + char buff[256]; + + logWarning("file: "__FILE__", line: %d, " \ + "trunk entry: %s", __LINE__, \ + trunk_info_dump(pTrunkInfo, buff, sizeof(buff))); + */ + + FILL_FILE_IDENTIFIER(target, pTrunkInfo); + + pFound = (FDFSTrunksById *)avl_tree_find(&tree_info_by_id, &target); + if (pFound == NULL) + { + return 0; + } + + /* + { + logWarning("file: "__FILE__", line: %d, " \ + "ARRAY COUNT: %d, trunk entry: %s", \ + __LINE__, pFound->block_array.count, \ + trunk_info_dump(pTrunkInfo, buff, sizeof(buff))); + } + */ + + if (pFound->block_array.count == 0) + { + return 0; + } + + blocks = pFound->block_array.blocks; + end_offset = pTrunkInfo->file.offset + pTrunkInfo->file.size; + if (end_offset <= blocks[0]->file.offset) + { + return 0; + } + + right = pFound->block_array.count - 1; + if (pTrunkInfo->file.offset >= blocks[right]->file.offset + \ + blocks[right]->file.size) + { + return 0; + } + + + result = 0; + mid = 0; + left = 0; + while (left <= right) + { + mid = (left + right) / 2; + if (pTrunkInfo->file.offset < blocks[mid]->file.offset) + { + if (blocks[mid]->file.offset < end_offset) + { + result = EEXIST; + break; + } + + right = mid - 1; + } + else if (pTrunkInfo->file.offset == blocks[mid]->file.offset) + { + if (pTrunkInfo->file.size == blocks[mid]->file.size) + { + char buff[256]; + logWarning("file: "__FILE__", line: %d, " \ + "node already exist, trunk entry: %s", \ + __LINE__, trunk_info_dump(pTrunkInfo, \ + buff, sizeof(buff))); + return EEXIST; + } + + result = EEXIST; + break; + } + else + { + if (pTrunkInfo->file.offset < (blocks[mid]->file.offset + \ + blocks[mid]->file.size)) + { + result = EEXIST; + break; + } + + left = mid + 1; + } + } + + if (result != 0) + { + char buff1[256]; + char buff2[256]; + + logWarning("file: "__FILE__", line: %d, " \ + "node overlap, current trunk entry: %s, " \ + "existed trunk entry: %s", __LINE__, \ + trunk_info_dump(pTrunkInfo, buff1, sizeof(buff1)), \ + trunk_info_dump(blocks[mid], buff2, sizeof(buff2))); + } + + return result; +} + +static int trunk_free_block_realloc(FDFSBlockArray *pArray, const int new_alloc) +{ + FDFSTrunkFullInfo **blocks; + int result; + + blocks = (FDFSTrunkFullInfo **)realloc(pArray->blocks, \ + new_alloc * sizeof(FDFSTrunkFullInfo *)); + if (blocks == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)(new_alloc * sizeof(FDFSTrunkFullInfo *)), + result, STRERROR(result)); + return result; + } + + pArray->alloc = new_alloc; + pArray->blocks = blocks; + return 0; +} + +static int trunk_free_block_do_insert(FDFSTrunkFullInfo *pTrunkInfo, \ + FDFSBlockArray *pArray) +{ + int left; + int right; + int mid; + int pos; + int result; + + if (pArray->count >= pArray->alloc) + { + if ((result=trunk_free_block_realloc(pArray, \ + pArray->alloc == 0 ? TRUNK_FREE_BLOCK_ARRAY_INIT_SIZE \ + : 2 * pArray->alloc)) != 0) + { + return result; + } + } + + if (pArray->count == 0) + { + pArray->blocks[pArray->count++] = pTrunkInfo; + return 0; + } + + if (pTrunkInfo->file.offset < pArray->blocks[0]->file.offset) + { + memmove(&(pArray->blocks[1]), &(pArray->blocks[0]), \ + pArray->count * sizeof(FDFSTrunkFullInfo *)); + pArray->blocks[0] = pTrunkInfo; + pArray->count++; + return 0; + } + + right = pArray->count - 1; + if (pTrunkInfo->file.offset > pArray->blocks[right]->file.offset) + { + pArray->blocks[pArray->count++] = pTrunkInfo; + return 0; + } + + left = 0; + mid = 0; + while (left <= right) + { + mid = (left + right) / 2; + if (pArray->blocks[mid]->file.offset > pTrunkInfo->file.offset) + { + right = mid - 1; + } + else if (pArray->blocks[mid]->file.offset == \ + pTrunkInfo->file.offset) + { + char buff[256]; + logWarning("file: "__FILE__", line: %d, " \ + "node already exist, trunk entry: %s", \ + __LINE__, trunk_info_dump(pTrunkInfo, \ + buff, sizeof(buff))); + return EEXIST; + } + else + { + left = mid + 1; + } + } + + if (pTrunkInfo->file.offset < pArray->blocks[mid]->file.offset) + { + pos = mid; + } + else + { + pos = mid + 1; + } + + memmove(&(pArray->blocks[pos + 1]), &(pArray->blocks[pos]), \ + (pArray->count - pos) * sizeof(FDFSTrunkFullInfo *)); + pArray->blocks[pos] = pTrunkInfo; + pArray->count++; + return 0; +} + +int trunk_free_block_insert(FDFSTrunkFullInfo *pTrunkInfo) +{ + int result; + FDFSTrunkFileIdentifier target; + FDFSTrunksById *pTrunksById; + + FILL_FILE_IDENTIFIER(target, pTrunkInfo); + + pTrunksById = (FDFSTrunksById *)avl_tree_find(&tree_info_by_id, &target); + if (pTrunksById == NULL) + { + pTrunksById = (FDFSTrunksById *)malloc(sizeof(FDFSTrunksById)); + if (pTrunksById == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDFSTrunksById), \ + result, STRERROR(result)); + return result; + } + + memset(pTrunksById, 0, sizeof(FDFSTrunksById)); + memcpy(&(pTrunksById->trunk_file_id), &target, \ + sizeof(FDFSTrunkFileIdentifier)); + if (avl_tree_insert(&tree_info_by_id, pTrunksById) != 1) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "avl_tree_insert fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + return trunk_free_block_do_insert(pTrunkInfo, \ + &(pTrunksById->block_array)); +} + +int trunk_free_block_delete(FDFSTrunkFullInfo *pTrunkInfo) +{ + int result; + int left; + int right; + int mid; + int move_count; + FDFSTrunkFileIdentifier target; + FDFSTrunksById *pTrunksById; + char buff[256]; + + FILL_FILE_IDENTIFIER(target, pTrunkInfo); + + pTrunksById = (FDFSTrunksById *)avl_tree_find(&tree_info_by_id, &target); + if (pTrunksById == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "node NOT exist, trunk entry: %s", \ + __LINE__, trunk_info_dump(pTrunkInfo, \ + buff, sizeof(buff))); + return ENOENT; + } + + result = ENOENT; + mid = 0; + left = 0; + right = pTrunksById->block_array.count - 1; + while (left <= right) + { + mid = (left + right) / 2; + if (pTrunksById->block_array.blocks[mid]->file.offset > \ + pTrunkInfo->file.offset) + { + right = mid - 1; + } + else if (pTrunksById->block_array.blocks[mid]->file.offset == \ + pTrunkInfo->file.offset) + { + result = 0; + break; + } + else + { + left = mid + 1; + } + } + + if (result == ENOENT) + { + logWarning("file: "__FILE__", line: %d, " \ + "trunk node NOT exist, trunk entry: %s", \ + __LINE__, trunk_info_dump(pTrunkInfo, \ + buff, sizeof(buff))); + return result; + } + + move_count = pTrunksById->block_array.count - (mid + 1); + if (move_count > 0) + { + memmove(&(pTrunksById->block_array.blocks[mid]), \ + &(pTrunksById->block_array.blocks[mid + 1]), \ + move_count * sizeof(FDFSTrunkFullInfo *)); + } + pTrunksById->block_array.count--; + + if (pTrunksById->block_array.count == 0) + { + free(pTrunksById->block_array.blocks); + if (avl_tree_delete(&tree_info_by_id, pTrunksById) != 1) + { + memset(&(pTrunksById->block_array), 0, \ + sizeof(FDFSBlockArray)); + logWarning("file: "__FILE__", line: %d, " \ + "can't delete block node, trunk info: %s", \ + __LINE__, trunk_info_dump(pTrunkInfo, buff, \ + sizeof(buff))); + return ENOENT; + } + } + else + { + if ((pTrunksById->block_array.count < \ + pTrunksById->block_array.alloc / 2) && \ + (pTrunksById->block_array.count > \ + TRUNK_FREE_BLOCK_ARRAY_INIT_SIZE / 2)) //small the array + { + if ((result=trunk_free_block_realloc( \ + &(pTrunksById->block_array), \ + pTrunksById->block_array.alloc / 2)) != 0) + { + return result; + } + } + } + + return 0; +} + +static int block_tree_print_walk_callback(void *data, void *args) +{ + FILE *fp; + FDFSBlockArray *pArray; + FDFSTrunkFullInfo **pp; + FDFSTrunkFullInfo **ppEnd; + + fp = (FILE *)args; + pArray = &(((FDFSTrunksById *)data)->block_array); + + /* + { + FDFSTrunkFileIdentifier *pFileIdentifier; + pFileIdentifier = &(((FDFSTrunksById *)data)->trunk_file_id); + + fprintf(fp, "%d %d %d %d %d", \ + pFileIdentifier->path.store_path_index, \ + pFileIdentifier->path.sub_path_high, \ + pFileIdentifier->path.sub_path_low, \ + pFileIdentifier->id, pArray->count); + if (pArray->count > 0) + { + fprintf(fp, " %d", pArray->blocks[0]->file.offset); + if (pArray->count > 1) + { + fprintf(fp, " %d", pArray->blocks[pArray->count-1]-> \ + file.offset + pArray->blocks[pArray->count-1]->\ + file.size); + } + } + fprintf(fp, "\n"); + return 0; + } + */ + + /* + { + FDFSTrunkFullInfo **ppPrevious; + if (pArray->count <= 1) + { + return 0; + } + ppPrevious = pArray->blocks; + ppEnd = pArray->blocks + pArray->count; + for (pp=pArray->blocks + 1; ppfile.offset >= (*pp)->file.offset) + { + fprintf(fp, "%d %d %d %d %d %d\n", \ + (*ppPrevious)->path.store_path_index, \ + (*ppPrevious)->path.sub_path_high, (*ppPrevious)->path.sub_path_low, \ + (*ppPrevious)->file.id, (*ppPrevious)->file.offset, (*ppPrevious)->file.size); + + fprintf(fp, "%d %d %d %d %d %d\n", \ + (*pp)->path.store_path_index, \ + (*pp)->path.sub_path_high, (*pp)->path.sub_path_low, \ + (*pp)->file.id, (*pp)->file.offset, (*pp)->file.size); + } + ppPrevious = pp; + } + return 0; + } + */ + + ppEnd = pArray->blocks + pArray->count; + for (pp=pArray->blocks; pppath.store_path_index, \ + (*pp)->path.sub_path_high, (*pp)->path.sub_path_low, \ + (*pp)->file.id, (*pp)->file.offset, (*pp)->file.size); + } + + return 0; +} + +int trunk_free_block_tree_print(const char *filename) +{ + FILE *fp; + int result; + + fp = fopen(filename, "w"); + if (fp == NULL) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + filename, result, STRERROR(result)); + return result; + } + + avl_tree_walk(&tree_info_by_id, block_tree_print_walk_callback, fp); + fclose(fp); + return 0; +} + diff --git a/storage/trunk_mgr/trunk_free_block_checker.h b/storage/trunk_mgr/trunk_free_block_checker.h new file mode 100644 index 0000000..663d9be --- /dev/null +++ b/storage/trunk_mgr/trunk_free_block_checker.h @@ -0,0 +1,60 @@ +/** +* Copyright (C) 2012 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_free_block_checker.h + +#ifndef _TRUNK_FREE_BLOCK_CHECKER_H_ +#define _TRUNK_FREE_BLOCK_CHECKER_H_ + +#include +#include +#include +#include +#include "common_define.h" +#include "fdfs_global.h" +#include "tracker_types.h" +#include "trunk_shared.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + FDFSTrunkPathInfo path; //trunk file path + int id; //trunk file id +} FDFSTrunkFileIdentifier; + +typedef struct { + int alloc; //alloc block count + int count; //block count + FDFSTrunkFullInfo **blocks; //sort by FDFSTrunkFullInfo.file.offset +} FDFSBlockArray; + +typedef struct { + FDFSTrunkFileIdentifier trunk_file_id; + FDFSBlockArray block_array; +} FDFSTrunksById; + +int trunk_free_block_checker_init(); +void trunk_free_block_checker_destroy(); + +int trunk_free_block_tree_node_count(); +int trunk_free_block_total_count(); + +int trunk_free_block_check_duplicate(FDFSTrunkFullInfo *pTrunkInfo); +int trunk_free_block_insert(FDFSTrunkFullInfo *pTrunkInfo); +int trunk_free_block_delete(FDFSTrunkFullInfo *pTrunkInfo); + +int trunk_free_block_tree_print(const char *filename); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/trunk_mgr/trunk_mem.c b/storage/trunk_mgr/trunk_mem.c new file mode 100644 index 0000000..7413bc6 --- /dev/null +++ b/storage/trunk_mgr/trunk_mem.c @@ -0,0 +1,1985 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_mem.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "avl_tree.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_service.h" +#include "trunk_sync.h" +#include "storage_dio.h" +#include "trunk_free_block_checker.h" +#include "trunk_mem.h" + +#define STORAGE_TRUNK_DATA_FILENAME "storage_trunk.dat" + +#define STORAGE_TRUNK_INIT_FLAG_NONE 0 +#define STORAGE_TRUNK_INIT_FLAG_DESTROYING 1 +#define STORAGE_TRUNK_INIT_FLAG_DONE 2 + +int g_slot_min_size; +int g_trunk_file_size; +int g_slot_max_size; +int g_store_path_mode = FDFS_STORE_PATH_ROUND_ROBIN; +FDFSStorageReservedSpace g_storage_reserved_space = { + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB}; +int g_avg_storage_reserved_mb = FDFS_DEF_STORAGE_RESERVED_MB; +int g_store_path_index = 0; +int g_current_trunk_file_id = 0; +TimeInfo g_trunk_create_file_time_base = {0, 0}; +int g_trunk_create_file_interval = 86400; +int g_trunk_compress_binlog_min_interval = 0; +ConnectionInfo g_trunk_server = {-1, 0}; +bool g_if_use_trunk_file = false; +bool g_if_trunker_self = false; +bool g_trunk_create_file_advance = false; +bool g_trunk_init_check_occupying = false; +bool g_trunk_init_reload_from_binlog = false; +static byte trunk_init_flag = STORAGE_TRUNK_INIT_FLAG_NONE; +int64_t g_trunk_total_free_space = 0; +int64_t g_trunk_create_file_space_threshold = 0; +time_t g_trunk_last_compress_time = 0; + +static pthread_mutex_t trunk_file_lock; +static pthread_mutex_t trunk_mem_lock; +static struct fast_mblock_man free_blocks_man; +static struct fast_mblock_man tree_nodes_man; + +static AVLTreeInfo *tree_info_by_sizes = NULL; //for block alloc + +static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo); +static int trunk_add_free_block(FDFSTrunkNode *pNode, const bool bWriteBinLog); + +static int trunk_restore_node(const FDFSTrunkFullInfo *pTrunkInfo); +static int trunk_delete_space(const FDFSTrunkFullInfo *pTrunkInfo, \ + const bool bWriteBinLog); + +static int storage_trunk_save(); +static int storage_trunk_load(); + +static int trunk_mem_binlog_write(const int timestamp, const char op_type, \ + const FDFSTrunkFullInfo *pTrunk) +{ + pthread_mutex_lock(&trunk_file_lock); + if (op_type == TRUNK_OP_TYPE_ADD_SPACE) + { + g_trunk_total_free_space += pTrunk->file.size; + } + else if (op_type == TRUNK_OP_TYPE_DEL_SPACE) + { + g_trunk_total_free_space -= pTrunk->file.size; + } + pthread_mutex_unlock(&trunk_file_lock); + + return trunk_binlog_write(timestamp, op_type, pTrunk); +} + +static int storage_trunk_node_compare_size(void *p1, void *p2) +{ + return ((FDFSTrunkSlot *)p1)->size - ((FDFSTrunkSlot *)p2)->size; +} + +static int storage_trunk_node_compare_offset(void *p1, void *p2) +{ + FDFSTrunkFullInfo *pTrunkInfo1; + FDFSTrunkFullInfo *pTrunkInfo2; + int result; + + pTrunkInfo1 = &(((FDFSTrunkNode *)p1)->trunk); + pTrunkInfo2 = &(((FDFSTrunkNode *)p2)->trunk); + + result = memcmp(&(pTrunkInfo1->path), &(pTrunkInfo2->path), \ + sizeof(FDFSTrunkPathInfo)); + if (result != 0) + { + return result; + } + + result = pTrunkInfo1->file.id - pTrunkInfo2->file.id; + if (result != 0) + { + return result; + } + + return pTrunkInfo1->file.offset - pTrunkInfo2->file.offset; +} + +char *storage_trunk_get_data_filename(char *full_filename) +{ + snprintf(full_filename, MAX_PATH_SIZE, "%s/data/%s", \ + g_fdfs_base_path, STORAGE_TRUNK_DATA_FILENAME); + return full_filename; +} + +#define STORAGE_TRUNK_CHECK_STATUS() \ + do \ + { \ + if (!g_if_trunker_self) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "I am not trunk server!", __LINE__); \ + return EINVAL; \ + } \ + if (trunk_init_flag != STORAGE_TRUNK_INIT_FLAG_DONE) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "I am not inited!", __LINE__); \ + return EINVAL; \ + } \ + } while (0) + +int storage_trunk_init() +{ + int result; + int i; + int count; + + if (!g_if_trunker_self) + { + logError("file: "__FILE__", line: %d, " \ + "I am not trunk server!", __LINE__); + return 0; + } + + if (trunk_init_flag != STORAGE_TRUNK_INIT_FLAG_NONE) + { + logWarning("file: "__FILE__", line: %d, " \ + "trunk already inited!", __LINE__); + return 0; + } + + logDebug("file: "__FILE__", line: %d, " \ + "storage trunk init ...", __LINE__); + + g_trunk_server.sock = -1; + g_trunk_server.port = g_server_port; + + if ((result=init_pthread_lock(&trunk_file_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if ((result=init_pthread_lock(&trunk_mem_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if ((result=fast_mblock_init(&free_blocks_man, \ + sizeof(FDFSTrunkNode), 0)) != 0) + { + return result; + } + + if ((result=fast_mblock_init(&tree_nodes_man, \ + sizeof(FDFSTrunkSlot), 0)) != 0) + { + return result; + } + + tree_info_by_sizes = (AVLTreeInfo *)malloc(sizeof(AVLTreeInfo) * \ + g_fdfs_store_paths.count); + if (tree_info_by_sizes == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)(sizeof(AVLTreeInfo) * \ + g_fdfs_store_paths.count), result, STRERROR(result)); + return result; + } + + for (i=0; ihead; + while (pCurrent != NULL) + { + pTrunkInfo = &pCurrent->trunk; + len = sprintf(pCallbackArgs->pCurrent, \ + "%d %c %d %d %d %d %d %d\n", \ + (int)g_current_time, TRUNK_OP_TYPE_ADD_SPACE, \ + pTrunkInfo->path.store_path_index, \ + pTrunkInfo->path.sub_path_high, \ + pTrunkInfo->path.sub_path_low, \ + pTrunkInfo->file.id, \ + pTrunkInfo->file.offset, \ + pTrunkInfo->file.size); + pCallbackArgs->pCurrent += len; + if (pCallbackArgs->pCurrent - pCallbackArgs->buff > \ + sizeof(pCallbackArgs->buff) - 128) + { + if (write(pCallbackArgs->fd, pCallbackArgs->buff, \ + pCallbackArgs->pCurrent - pCallbackArgs->buff) \ + != pCallbackArgs->pCurrent - pCallbackArgs->buff) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, "\ + "write to file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pCallbackArgs->temp_trunk_filename, \ + result, STRERROR(result)); + return result; + } + + pCallbackArgs->pCurrent = pCallbackArgs->buff; + } + + pCurrent = pCurrent->next; + } + + return 0; +} + +static int storage_trunk_do_save() +{ + int64_t trunk_binlog_size; + char trunk_data_filename[MAX_PATH_SIZE]; + struct walk_callback_args callback_args; + int len; + int result; + int i; + + trunk_binlog_size = storage_trunk_get_binlog_size(); + if (trunk_binlog_size < 0) + { + return errno != 0 ? errno : EPERM; + } + + memset(&callback_args, 0, sizeof(callback_args)); + callback_args.pCurrent = callback_args.buff; + + sprintf(callback_args.temp_trunk_filename, "%s/data/.%s.tmp", \ + g_fdfs_base_path, STORAGE_TRUNK_DATA_FILENAME); + callback_args.fd = open(callback_args.temp_trunk_filename, \ + O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (callback_args.fd < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, callback_args.temp_trunk_filename, \ + result, STRERROR(result)); + return result; + } + + len = sprintf(callback_args.pCurrent, INT64_PRINTF_FORMAT"\n", \ + trunk_binlog_size); + callback_args.pCurrent += len; + + result = 0; + pthread_mutex_lock(&trunk_mem_lock); + for (i=0; i 0 && result == 0) + { + if (write(callback_args.fd, callback_args.buff, len) != len) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, "\ + "write to file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, callback_args.temp_trunk_filename, \ + result, STRERROR(result)); + } + } + + if (result == 0 && fsync(callback_args.fd) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, "\ + "fsync file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, callback_args.temp_trunk_filename, \ + result, STRERROR(result)); + } + + if (close(callback_args.fd) != 0) + { + if (result == 0) + { + result = errno != 0 ? errno : EIO; + } + logError("file: "__FILE__", line: %d, "\ + "close file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, callback_args.temp_trunk_filename, \ + errno, STRERROR(errno)); + } + pthread_mutex_unlock(&trunk_mem_lock); + + if (result != 0) + { + return result; + } + + storage_trunk_get_data_filename(trunk_data_filename); + if (rename(callback_args.temp_trunk_filename, trunk_data_filename) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, "\ + "rename file %s to %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + callback_args.temp_trunk_filename, trunk_data_filename, \ + result, STRERROR(result)); + } + + return result; +} + +static int storage_trunk_save() +{ + int result; + + if (!(g_trunk_compress_binlog_min_interval > 0 && \ + g_current_time - g_trunk_last_compress_time > + g_trunk_compress_binlog_min_interval)) + { + return storage_trunk_do_save(); + } + + logInfo("start compress trunk binlog ..."); + if ((result=trunk_binlog_compress_apply()) != 0) + { + return result; + } + + if ((result=storage_trunk_do_save()) != 0) + { + trunk_binlog_compress_rollback(); + return result; + } + + if ((result=trunk_binlog_compress_commit()) != 0) + { + trunk_binlog_compress_rollback(); + return result; + } + + g_trunk_last_compress_time = g_current_time; + storage_write_to_sync_ini_file(); + + logInfo("compress trunk binlog done."); + return trunk_unlink_all_mark_files(); //because the binlog file be compressed +} + +static bool storage_trunk_is_space_occupied(const FDFSTrunkFullInfo *pTrunkInfo) +{ + int result; + int fd; + char full_filename[MAX_PATH_SIZE+64]; + + trunk_get_full_filename(pTrunkInfo, full_filename, sizeof(full_filename)); + fd = open(full_filename, O_RDONLY, 0644); + if (fd < 0) + { + result = errno != 0 ? errno : ENOENT; + logWarning("file: " __FILE__ ", line: %d, " + "open file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + result, STRERROR(result)); + return false; + } + + if (pTrunkInfo->file.offset > 0 && lseek(fd, pTrunkInfo->file.offset, \ + SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "lseek file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + result, STRERROR(result)); + close(fd); + return false; + } + + /* + logInfo("fd: %d, trunk filename: %s, offset: %d", fd, full_filename, \ + pTrunkInfo->file.offset); + */ + result = dio_check_trunk_file_ex(fd, full_filename, \ + pTrunkInfo->file.offset); + close(fd); + return (result == EEXIST); +} + +static int trunk_add_space_by_trunk(const FDFSTrunkFullInfo *pTrunkInfo) +{ + int result; + + result = trunk_free_space(pTrunkInfo, false); + if (result == 0 || result == EEXIST) + { + return 0; + } + else + { + return result; + } +} + +static int trunk_add_space_by_node(FDFSTrunkNode *pTrunkNode) +{ + int result; + + if (pTrunkNode->trunk.file.size < g_slot_min_size) + { + logDebug("file: "__FILE__", line: %d, " \ + "space: %d is too small, do not need recycle!", \ + __LINE__, pTrunkNode->trunk.file.size); + fast_mblock_free(&free_blocks_man, pTrunkNode->pMblockNode); + return 0; + } + + result = trunk_add_free_block(pTrunkNode, false); + if (result == 0) + { + return 0; + } + else + { + fast_mblock_free(&free_blocks_man, pTrunkNode->pMblockNode); + return (result == EEXIST) ? 0 : result; + } +} + +static int storage_trunk_do_add_space(const FDFSTrunkFullInfo *pTrunkInfo) +{ + if (g_trunk_init_check_occupying) + { + if (storage_trunk_is_space_occupied(pTrunkInfo)) + { + return 0; + } + } + + /* + { + char buff[256]; + trunk_info_dump(pTrunkInfo, buff, sizeof(buff)); + logInfo("add trunk info: %s", buff); + } + */ + + return trunk_add_space_by_trunk(pTrunkInfo); +} + +static void storage_trunk_free_node(void *ptr) +{ + fast_mblock_free(&free_blocks_man, \ + ((FDFSTrunkNode *)ptr)->pMblockNode); +} + +static int storage_trunk_add_free_blocks_callback(void *data, void *args) +{ + /* + char buff[256]; + logInfo("file: "__FILE__", line: %d"\ + ", adding trunk info: %s", __LINE__, \ + trunk_info_dump(&(((FDFSTrunkNode *)data)->trunk), \ + buff, sizeof(buff))); + */ + return trunk_add_space_by_node((FDFSTrunkNode *)data); +} + +static int storage_trunk_restore(const int64_t restore_offset) +{ + int64_t trunk_binlog_size; + int64_t line_count; + TrunkBinLogReader reader; + TrunkBinLogRecord record; + char trunk_mark_filename[MAX_PATH_SIZE]; + char buff[256]; + int record_length; + int result; + AVLTreeInfo tree_info_by_offset; + struct fast_mblock_node *pMblockNode; + FDFSTrunkNode *pTrunkNode; + FDFSTrunkNode trunkNode; + bool trunk_init_reload_from_binlog; + + trunk_binlog_size = storage_trunk_get_binlog_size(); + if (trunk_binlog_size < 0) + { + return errno != 0 ? errno : EPERM; + } + + if (restore_offset == trunk_binlog_size) + { + return 0; + } + + if (restore_offset > trunk_binlog_size) + { + logWarning("file: "__FILE__", line: %d, " \ + "restore_offset: "INT64_PRINTF_FORMAT \ + " > trunk_binlog_size: "INT64_PRINTF_FORMAT, \ + __LINE__, restore_offset, trunk_binlog_size); + return storage_trunk_save(); + } + + logDebug("file: "__FILE__", line: %d, " \ + "trunk metadata recovering, start offset: " \ + INT64_PRINTF_FORMAT", need recovery binlog bytes: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + restore_offset, trunk_binlog_size - restore_offset); + + trunk_init_reload_from_binlog = (restore_offset == 0); + if (trunk_init_reload_from_binlog) + { + memset(&trunkNode, 0, sizeof(trunkNode)); + if ((result=avl_tree_init(&tree_info_by_offset, \ + storage_trunk_free_node, \ + storage_trunk_node_compare_offset)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "avl_tree_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + } + + memset(&record, 0, sizeof(record)); + memset(&reader, 0, sizeof(reader)); + reader.binlog_offset = restore_offset; + if ((result=trunk_reader_init(NULL, &reader)) != 0) + { + return result; + } + + line_count = 0; + while (1) + { + record_length = 0; + result = trunk_binlog_read(&reader, &record, &record_length); + if (result != 0) + { + if (result == ENOENT) + { + if (record_length > 0) //skip incorrect record + { + line_count++; + reader.binlog_offset += record_length; + continue; + } + + result = (reader.binlog_offset >= \ + trunk_binlog_size) ? 0 : EINVAL; + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "binlog offset: "INT64_PRINTF_FORMAT \ + " < binlog size: "INT64_PRINTF_FORMAT \ + ", please check the end of trunk " \ + "binlog", __LINE__, \ + reader.binlog_offset, trunk_binlog_size); + } + } + + break; + } + + line_count++; + if (record.op_type == TRUNK_OP_TYPE_ADD_SPACE) + { + record.trunk.status = FDFS_TRUNK_STATUS_FREE; + + if (trunk_init_reload_from_binlog) + { + pMblockNode = fast_mblock_alloc(&free_blocks_man); + if (pMblockNode == NULL) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, "\ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, \ + (int)sizeof(FDFSTrunkNode), \ + result, STRERROR(result)); + return result; + } + + pTrunkNode = (FDFSTrunkNode *)pMblockNode->data; + memcpy(&pTrunkNode->trunk, &(record.trunk), \ + sizeof(FDFSTrunkFullInfo)); + + pTrunkNode->pMblockNode = pMblockNode; + pTrunkNode->next = NULL; + result = avl_tree_insert(&tree_info_by_offset,\ + pTrunkNode); + if (result < 0) //error + { + result *= -1; + logError("file: "__FILE__", line: %d, "\ + "avl_tree_insert fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, \ + STRERROR(result)); + return result; + } + else if (result == 0) + { + trunk_info_dump(&(record.trunk), \ + buff, sizeof(buff)); + logWarning("file: "__FILE__", line: %d"\ + ", trunk data line: " \ + INT64_PRINTF_FORMAT", trunk "\ + "space already exist, "\ + "trunk info: %s", \ + __LINE__, line_count, buff); + } + } + else if ((result=trunk_add_space_by_trunk( \ + &record.trunk)) != 0) + { + break; + } + } + else if (record.op_type == TRUNK_OP_TYPE_DEL_SPACE) + { + record.trunk.status = FDFS_TRUNK_STATUS_FREE; + if (trunk_init_reload_from_binlog) + { + memcpy(&trunkNode.trunk, &record.trunk, \ + sizeof(FDFSTrunkFullInfo)); + if (avl_tree_delete(&tree_info_by_offset,\ + &trunkNode) != 1) + { + trunk_info_dump(&(record.trunk), \ + buff, sizeof(buff)); + logWarning("file: "__FILE__", line: %d"\ + ", binlog offset: "INT64_PRINTF_FORMAT \ + ", trunk data line: "INT64_PRINTF_FORMAT \ + " trunk node not exist, " \ + "trunk info: %s", __LINE__, \ + reader.binlog_offset, \ + line_count, buff); + } + } + else if ((result=trunk_delete_space( \ + &record.trunk, false)) != 0) + { + if (result == ENOENT) + { + logDebug("file: "__FILE__", line: %d, "\ + "binlog offset: "INT64_PRINTF_FORMAT \ + ", trunk data line: "INT64_PRINTF_FORMAT,\ + __LINE__, reader.binlog_offset, \ + line_count); + + result = 0; + } + else + { + break; + } + } + } + + reader.binlog_offset += record_length; + } + + trunk_reader_destroy(&reader); + trunk_mark_filename_by_reader(&reader, trunk_mark_filename); + if (unlink(trunk_mark_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "unlink file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + trunk_mark_filename, errno, STRERROR(errno)); + } + + if (result != 0) + { + if (trunk_init_reload_from_binlog) + { + avl_tree_destroy(&tree_info_by_offset); + } + + logError("file: "__FILE__", line: %d, " \ + "trunk load fail, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if (trunk_init_reload_from_binlog) + { + logInfo("file: "__FILE__", line: %d, " \ + "free tree node count: %d", \ + __LINE__, avl_tree_count(&tree_info_by_offset)); + + result = avl_tree_walk(&tree_info_by_offset, \ + storage_trunk_add_free_blocks_callback, NULL); + + tree_info_by_offset.free_data_func = NULL; + avl_tree_destroy(&tree_info_by_offset); + } + + if (result == 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "trunk metadata recovery done. start offset: " \ + INT64_PRINTF_FORMAT", recovery file size: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + restore_offset, trunk_binlog_size - restore_offset); + return storage_trunk_save(); + } + + return result; +} + +int storage_delete_trunk_data_file() +{ + char trunk_data_filename[MAX_PATH_SIZE]; + int result; + + storage_trunk_get_data_filename(trunk_data_filename); + if (unlink(trunk_data_filename) == 0) + { + return 0; + } + + result = errno != 0 ? errno : ENOENT; + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "unlink trunk data file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, trunk_data_filename, \ + result, STRERROR(result)); + } + + return result; +} + +static int storage_trunk_load() +{ +#define TRUNK_DATA_NEW_FIELD_COUNT 8 // >= v5.01 +#define TRUNK_DATA_OLD_FIELD_COUNT 6 // < V5.01 +#define TRUNK_LINE_MAX_LENGHT 64 + + int64_t restore_offset; + char trunk_data_filename[MAX_PATH_SIZE]; + char buff[4 * 1024 + 1]; + int line_count; + int col_count; + int index; + char *pLineStart; + char *pLineEnd; + char *cols[TRUNK_DATA_NEW_FIELD_COUNT]; + FDFSTrunkFullInfo trunkInfo; + int result; + int fd; + int bytes; + int len; + + storage_trunk_get_data_filename(trunk_data_filename); + if (g_trunk_init_reload_from_binlog) + { + if (unlink(trunk_data_filename) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "unlink file %s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + trunk_data_filename, result, \ + STRERROR(result)); + return result; + } + } + + restore_offset = 0; + return storage_trunk_restore(restore_offset); + } + + fd = open(trunk_data_filename, O_RDONLY); + if (fd < 0) + { + result = errno != 0 ? errno : EIO; + if (result == ENOENT) + { + restore_offset = 0; + return storage_trunk_restore(restore_offset); + } + + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, trunk_data_filename, \ + result, STRERROR(result)); + return result; + } + + if ((bytes=read(fd, buff, sizeof(buff) - 1)) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "read from file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, trunk_data_filename, \ + result, STRERROR(result)); + close(fd); + return result; + } + + *(buff + bytes) = '\0'; + pLineEnd = strchr(buff, '\n'); + if (pLineEnd == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "read offset from file %s fail", \ + __LINE__, trunk_data_filename); + close(fd); + return EINVAL; + } + + *pLineEnd = '\0'; + restore_offset = strtoll(buff, NULL, 10); + pLineStart = pLineEnd + 1; //skip \n + line_count = 0; + while (1) + { + pLineEnd = strchr(pLineStart, '\n'); + if (pLineEnd == NULL) + { + if (bytes < sizeof(buff) - 1) //EOF + { + break; + } + + len = strlen(pLineStart); + if (len > TRUNK_LINE_MAX_LENGHT) + { + logError("file: "__FILE__", line: %d, " \ + "file %s, line length: %d too long", \ + __LINE__, trunk_data_filename, len); + close(fd); + return EINVAL; + } + + memcpy(buff, pLineStart, len); + if ((bytes=read(fd, buff + len, sizeof(buff) \ + - len - 1)) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "read from file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, trunk_data_filename, \ + result, STRERROR(result)); + close(fd); + return result; + } + + if (bytes == 0) + { + result = ENOENT; + logError("file: "__FILE__", line: %d, " \ + "file: %s, end of file, expect " \ + "end line", __LINE__, \ + trunk_data_filename); + close(fd); + return result; + } + + bytes += len; + *(buff + bytes) = '\0'; + pLineStart = buff; + continue; + } + + ++line_count; + *pLineEnd = '\0'; + col_count = splitEx(pLineStart, ' ', cols, + TRUNK_DATA_NEW_FIELD_COUNT); + if (col_count != TRUNK_DATA_NEW_FIELD_COUNT && \ + col_count != TRUNK_DATA_OLD_FIELD_COUNT) + { + logError("file: "__FILE__", line: %d, " \ + "file %s, line: %d is invalid", \ + __LINE__, trunk_data_filename, line_count); + close(fd); + return EINVAL; + } + + if (col_count == TRUNK_DATA_OLD_FIELD_COUNT) + { + index = 0; + } + else + { + index = 2; + } + trunkInfo.path.store_path_index = atoi(cols[index++]); + trunkInfo.path.sub_path_high = atoi(cols[index++]); + trunkInfo.path.sub_path_low = atoi(cols[index++]); + trunkInfo.file.id = atoi(cols[index++]); + trunkInfo.file.offset = atoi(cols[index++]); + trunkInfo.file.size = atoi(cols[index++]); + if ((result=storage_trunk_do_add_space(&trunkInfo)) != 0) + { + close(fd); + return result; + } + + pLineStart = pLineEnd + 1; //next line + } + + close(fd); + + if (*pLineStart != '\0') + { + logError("file: "__FILE__", line: %d, " \ + "file %s does not end correctly", \ + __LINE__, trunk_data_filename); + return EINVAL; + } + + logDebug("file: "__FILE__", line: %d, " \ + "file %s, line count: %d", \ + __LINE__, trunk_data_filename, line_count); + + return storage_trunk_restore(restore_offset); +} + +int trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo, \ + const bool bWriteBinLog) +{ + int result; + struct fast_mblock_node *pMblockNode; + FDFSTrunkNode *pTrunkNode; + + if (!g_if_trunker_self) + { + logError("file: "__FILE__", line: %d, " \ + "I am not trunk server!", __LINE__); + return EINVAL; + } + + if (trunk_init_flag != STORAGE_TRUNK_INIT_FLAG_DONE) + { + if (bWriteBinLog) + { + logError("file: "__FILE__", line: %d, " \ + "I am not inited!", __LINE__); + return EINVAL; + } + } + + if (pTrunkInfo->file.size < g_slot_min_size) + { + logDebug("file: "__FILE__", line: %d, " \ + "space: %d is too small, do not need recycle!", \ + __LINE__, pTrunkInfo->file.size); + return 0; + } + + pMblockNode = fast_mblock_alloc(&free_blocks_man); + if (pMblockNode == NULL) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDFSTrunkNode), \ + result, STRERROR(result)); + return result; + } + + pTrunkNode = (FDFSTrunkNode *)pMblockNode->data; + memcpy(&pTrunkNode->trunk, pTrunkInfo, sizeof(FDFSTrunkFullInfo)); + + pTrunkNode->pMblockNode = pMblockNode; + pTrunkNode->trunk.status = FDFS_TRUNK_STATUS_FREE; + pTrunkNode->next = NULL; + return trunk_add_free_block(pTrunkNode, bWriteBinLog); +} + +static int trunk_add_free_block(FDFSTrunkNode *pNode, const bool bWriteBinLog) +{ + int result; + struct fast_mblock_node *pMblockNode; + FDFSTrunkSlot target_slot; + FDFSTrunkSlot *chain; + + pthread_mutex_lock(&trunk_mem_lock); + + if ((result=trunk_free_block_check_duplicate(&(pNode->trunk))) != 0) + { + pthread_mutex_unlock(&trunk_mem_lock); + return result; + } + + target_slot.size = pNode->trunk.file.size; + target_slot.head = NULL; + chain = (FDFSTrunkSlot *)avl_tree_find(tree_info_by_sizes + \ + pNode->trunk.path.store_path_index, &target_slot); + if (chain == NULL) + { + pMblockNode = fast_mblock_alloc(&tree_nodes_man); + if (pMblockNode == NULL) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDFSTrunkSlot), \ + result, STRERROR(result)); + pthread_mutex_unlock(&trunk_mem_lock); + return result; + } + + chain = (FDFSTrunkSlot *)pMblockNode->data; + chain->pMblockNode = pMblockNode; + chain->size = pNode->trunk.file.size; + pNode->next = NULL; + chain->head = pNode; + + if (avl_tree_insert(tree_info_by_sizes + pNode->trunk. \ + path.store_path_index, chain) != 1) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "avl_tree_insert fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + pthread_mutex_unlock(&trunk_mem_lock); + return result; + } + } + else + { + pNode->next = chain->head; + chain->head = pNode; + } + + if (bWriteBinLog) + { + result = trunk_mem_binlog_write(g_current_time, \ + TRUNK_OP_TYPE_ADD_SPACE, &(pNode->trunk)); + } + else + { + pthread_mutex_lock(&trunk_file_lock); + g_trunk_total_free_space += pNode->trunk.file.size; + pthread_mutex_unlock(&trunk_file_lock); + result = 0; + } + + if (result == 0) + { + result = trunk_free_block_insert(&(pNode->trunk)); + } + else + { + trunk_free_block_insert(&(pNode->trunk)); + } + + pthread_mutex_unlock(&trunk_mem_lock); + + return result; +} + +static void trunk_delete_size_tree_entry(const int store_path_index, \ + FDFSTrunkSlot *pSlot) +{ + if (avl_tree_delete(tree_info_by_sizes + store_path_index, pSlot) == 1) + { + fast_mblock_free(&tree_nodes_man, \ + pSlot->pMblockNode); + } + else + { + logWarning("file: "__FILE__", line: %d, " \ + "can't delete slot entry, size: %d", \ + __LINE__, pSlot->size); + } +} + +static int trunk_delete_space(const FDFSTrunkFullInfo *pTrunkInfo, \ + const bool bWriteBinLog) +{ + int result; + FDFSTrunkSlot target_slot; + char buff[256]; + FDFSTrunkSlot *pSlot; + FDFSTrunkNode *pPrevious; + FDFSTrunkNode *pCurrent; + + target_slot.size = pTrunkInfo->file.size; + target_slot.head = NULL; + + pthread_mutex_lock(&trunk_mem_lock); + pSlot = (FDFSTrunkSlot *)avl_tree_find(tree_info_by_sizes + \ + pTrunkInfo->path.store_path_index, &target_slot); + if (pSlot == NULL) + { + pthread_mutex_unlock(&trunk_mem_lock); + logError("file: "__FILE__", line: %d, " \ + "can't find trunk entry: %s", __LINE__, \ + trunk_info_dump(pTrunkInfo, buff, sizeof(buff))); + return ENOENT; + } + + pPrevious = NULL; + pCurrent = pSlot->head; + while (pCurrent != NULL && memcmp(&(pCurrent->trunk), pTrunkInfo, \ + sizeof(FDFSTrunkFullInfo)) != 0) + { + pPrevious = pCurrent; + pCurrent = pCurrent->next; + } + + if (pCurrent == NULL) + { + pthread_mutex_unlock(&trunk_mem_lock); + logError("file: "__FILE__", line: %d, " \ + "can't find trunk entry: %s", __LINE__, \ + trunk_info_dump(pTrunkInfo, buff, sizeof(buff))); + return ENOENT; + } + + if (pPrevious == NULL) + { + pSlot->head = pCurrent->next; + if (pSlot->head == NULL) + { + trunk_delete_size_tree_entry(pTrunkInfo->path. \ + store_path_index, pSlot); + } + } + else + { + pPrevious->next = pCurrent->next; + } + + trunk_free_block_delete(&(pCurrent->trunk)); + pthread_mutex_unlock(&trunk_mem_lock); + + if (bWriteBinLog) + { + result = trunk_mem_binlog_write(g_current_time, \ + TRUNK_OP_TYPE_DEL_SPACE, &(pCurrent->trunk)); + } + else + { + pthread_mutex_lock(&trunk_file_lock); + g_trunk_total_free_space -= pCurrent->trunk.file.size; + pthread_mutex_unlock(&trunk_file_lock); + result = 0; + } + + fast_mblock_free(&free_blocks_man, pCurrent->pMblockNode); + return result; +} + +static int trunk_restore_node(const FDFSTrunkFullInfo *pTrunkInfo) +{ + FDFSTrunkSlot target_slot; + char buff[256]; + FDFSTrunkSlot *pSlot; + FDFSTrunkNode *pCurrent; + + target_slot.size = pTrunkInfo->file.size; + target_slot.head = NULL; + + pthread_mutex_lock(&trunk_mem_lock); + pSlot = (FDFSTrunkSlot *)avl_tree_find(tree_info_by_sizes + \ + pTrunkInfo->path.store_path_index, &target_slot); + if (pSlot == NULL) + { + pthread_mutex_unlock(&trunk_mem_lock); + logError("file: "__FILE__", line: %d, " \ + "can't find trunk entry: %s", __LINE__, \ + trunk_info_dump(pTrunkInfo, buff, sizeof(buff))); + return ENOENT; + } + + pCurrent = pSlot->head; + while (pCurrent != NULL && memcmp(&(pCurrent->trunk), \ + pTrunkInfo, sizeof(FDFSTrunkFullInfo)) != 0) + { + pCurrent = pCurrent->next; + } + + if (pCurrent == NULL) + { + pthread_mutex_unlock(&trunk_mem_lock); + + logError("file: "__FILE__", line: %d, " \ + "can't find trunk entry: %s", __LINE__, \ + trunk_info_dump(pTrunkInfo, buff, sizeof(buff))); + return ENOENT; + } + + pCurrent->trunk.status = FDFS_TRUNK_STATUS_FREE; + pthread_mutex_unlock(&trunk_mem_lock); + + return 0; +} + +static int trunk_split(FDFSTrunkNode *pNode, const int size) +{ + int result; + struct fast_mblock_node *pMblockNode; + FDFSTrunkNode *pTrunkNode; + + if (pNode->trunk.file.size - size < g_slot_min_size) + { + return trunk_mem_binlog_write(g_current_time, \ + TRUNK_OP_TYPE_DEL_SPACE, &(pNode->trunk)); + } + + pMblockNode = fast_mblock_alloc(&free_blocks_man); + if (pMblockNode == NULL) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDFSTrunkNode), \ + result, STRERROR(result)); + return result; + } + + result = trunk_mem_binlog_write(g_current_time, \ + TRUNK_OP_TYPE_DEL_SPACE, &(pNode->trunk)); + if (result != 0) + { + fast_mblock_free(&free_blocks_man, pMblockNode); + return result; + } + + pTrunkNode = (FDFSTrunkNode *)pMblockNode->data; + memcpy(pTrunkNode, pNode, sizeof(FDFSTrunkNode)); + + pTrunkNode->pMblockNode = pMblockNode; + pTrunkNode->trunk.file.offset = pNode->trunk.file.offset + size; + pTrunkNode->trunk.file.size = pNode->trunk.file.size - size; + pTrunkNode->trunk.status = FDFS_TRUNK_STATUS_FREE; + pTrunkNode->next = NULL; + + result = trunk_add_free_block(pTrunkNode, true); + if (result != 0) + { + return result; + } + + pNode->trunk.file.size = size; + return 0; +} + +static FDFSTrunkNode *trunk_create_trunk_file(const int store_path_index, \ + int *err_no) +{ + FDFSTrunkNode *pTrunkNode; + struct fast_mblock_node *pMblockNode; + + pMblockNode = fast_mblock_alloc(&free_blocks_man); + if (pMblockNode == NULL) + { + *err_no = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(FDFSTrunkNode), \ + *err_no, STRERROR(*err_no)); + return NULL; + } + + pTrunkNode = (FDFSTrunkNode *)pMblockNode->data; + pTrunkNode->pMblockNode = pMblockNode; + + if (store_path_index >= 0) + { + pTrunkNode->trunk.path.store_path_index = store_path_index; + } + else + { + int result; + int new_store_path_index; + if ((result=storage_get_storage_path_index( \ + &new_store_path_index)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "get_storage_path_index fail, " \ + "errno: %d, error info: %s", __LINE__, \ + result, STRERROR(result)); + return NULL; + } + pTrunkNode->trunk.path.store_path_index = new_store_path_index; + } + + pTrunkNode->trunk.file.offset = 0; + pTrunkNode->trunk.file.size = g_trunk_file_size; + pTrunkNode->trunk.status = FDFS_TRUNK_STATUS_FREE; + pTrunkNode->next = NULL; + + *err_no = trunk_create_next_file(&(pTrunkNode->trunk)); + if (*err_no != 0) + { + fast_mblock_free(&free_blocks_man, pMblockNode); + return NULL; + } + + *err_no = trunk_mem_binlog_write(g_current_time, \ + TRUNK_OP_TYPE_ADD_SPACE, &(pTrunkNode->trunk)); + return pTrunkNode; +} + +int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult) +{ + FDFSTrunkSlot target_slot; + FDFSTrunkSlot *pSlot; + FDFSTrunkNode *pPreviousNode; + FDFSTrunkNode *pTrunkNode; + int result; + + STORAGE_TRUNK_CHECK_STATUS(); + + target_slot.size = (size > g_slot_min_size) ? size : g_slot_min_size; + target_slot.head = NULL; + + pPreviousNode = NULL; + pTrunkNode = NULL; + pthread_mutex_lock(&trunk_mem_lock); + while (1) + { + pSlot = (FDFSTrunkSlot *)avl_tree_find_ge(tree_info_by_sizes \ + + pResult->path.store_path_index, &target_slot); + if (pSlot == NULL) + { + break; + } + + pPreviousNode = NULL; + pTrunkNode = pSlot->head; + while (pTrunkNode != NULL && \ + pTrunkNode->trunk.status == FDFS_TRUNK_STATUS_HOLD) + { + pPreviousNode = pTrunkNode; + pTrunkNode = pTrunkNode->next; + } + + if (pTrunkNode != NULL) + { + break; + } + + target_slot.size = pSlot->size + 1; + } + + if (pTrunkNode != NULL) + { + if (pPreviousNode == NULL) + { + pSlot->head = pTrunkNode->next; + if (pSlot->head == NULL) + { + trunk_delete_size_tree_entry(pResult->path. \ + store_path_index, pSlot); + } + } + else + { + pPreviousNode->next = pTrunkNode->next; + } + + trunk_free_block_delete(&(pTrunkNode->trunk)); + } + else + { + pTrunkNode = trunk_create_trunk_file(pResult->path. \ + store_path_index, &result); + if (pTrunkNode == NULL) + { + pthread_mutex_unlock(&trunk_mem_lock); + return result; + } + } + pthread_mutex_unlock(&trunk_mem_lock); + + result = trunk_split(pTrunkNode, size); + if (result != 0) + { + return result; + } + + pTrunkNode->trunk.status = FDFS_TRUNK_STATUS_HOLD; + result = trunk_add_free_block(pTrunkNode, true); + if (result == 0) + { + memcpy(pResult, &(pTrunkNode->trunk), \ + sizeof(FDFSTrunkFullInfo)); + } + + return result; +} + +int trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, const int status) +{ + FDFSTrunkFullInfo target_trunk_info; + + STORAGE_TRUNK_CHECK_STATUS(); + + memset(&target_trunk_info, 0, sizeof(FDFSTrunkFullInfo)); + target_trunk_info.status = FDFS_TRUNK_STATUS_HOLD; + target_trunk_info.path.store_path_index = \ + pTrunkInfo->path.store_path_index; + target_trunk_info.path.sub_path_high = pTrunkInfo->path.sub_path_high; + target_trunk_info.path.sub_path_low = pTrunkInfo->path.sub_path_low; + target_trunk_info.file.id = pTrunkInfo->file.id; + target_trunk_info.file.offset = pTrunkInfo->file.offset; + target_trunk_info.file.size = pTrunkInfo->file.size; + + if (status == 0) + { + return trunk_delete_space(&target_trunk_info, true); + } + else if (status == EEXIST) + { + char buff[256]; + trunk_info_dump(&target_trunk_info, buff, sizeof(buff)); + + logWarning("file: "__FILE__", line: %d, " \ + "trunk space already be occupied, " \ + "delete this trunk space, trunk info: %s", \ + __LINE__, buff); + return trunk_delete_space(&target_trunk_info, true); + } + else + { + return trunk_restore_node(&target_trunk_info); + } +} + +static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo) +{ + char buff[32]; + int result; + int filename_len; + char short_filename[64]; + char full_filename[MAX_PATH_SIZE]; + int sub_path_high; + int sub_path_low; + + while (1) + { + pthread_mutex_lock(&trunk_file_lock); + pTrunkInfo->file.id = ++g_current_trunk_file_id; + result = storage_write_to_sync_ini_file(); + pthread_mutex_unlock(&trunk_file_lock); + if (result != 0) + { + return result; + } + + int2buff(pTrunkInfo->file.id, buff); + base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(int), \ + short_filename, &filename_len, false); + + storage_get_store_path(short_filename, filename_len, \ + &sub_path_high, &sub_path_low); + + pTrunkInfo->path.sub_path_high = sub_path_high; + pTrunkInfo->path.sub_path_low = sub_path_low; + + trunk_get_full_filename(pTrunkInfo, full_filename, \ + sizeof(full_filename)); + if (!fileExists(full_filename)) + { + break; + } + } + + if ((result=trunk_init_file(full_filename)) != 0) + { + return result; + } + + return 0; +} + +static int trunk_wait_file_ready(const char *filename, const int64_t file_size, + const bool log_when_no_ent) +{ + struct stat file_stat; + time_t file_mtime; + int result; + + if (stat(filename, &file_stat) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (log_when_no_ent || result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "stat file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + } + return result; + } + + file_mtime = file_stat.st_mtime; + while (1) + { + if (file_stat.st_size >= file_size) + { + return 0; + } + + if (abs(g_current_time - file_mtime) > 10) + { + return ETIMEDOUT; + } + + usleep(5 * 1000); + + if (stat(filename, &file_stat) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (log_when_no_ent || result != ENOENT) + { + logError("file: "__FILE__", line: %d, " \ + "stat file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + } + return result; + } + } + + return 0; +} + +int trunk_init_file_ex(const char *filename, const int64_t file_size) +{ + int fd; + int result; + + fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0644); + if (fd < 0) + { + result = errno != 0 ? errno : EEXIST; + if (result == EEXIST) //already created by another dio thread + { + logDebug("file: "__FILE__", line: %d, " \ + "waiting for trunk file: %s " \ + "ready ...", __LINE__, filename); + + result = trunk_wait_file_ready(filename, file_size, true); + if (result == ETIMEDOUT) + { + logError("file: "__FILE__", line: %d, " \ + "waiting for trunk file: %s " \ + "ready timeout!", __LINE__, filename); + } + + logDebug("file: "__FILE__", line: %d, " \ + "waiting for trunk file: %s " \ + "done.", __LINE__, filename); + return result; + } + + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + return result; + } + + if (ftruncate(fd, file_size) == 0) + { + result = 0; + } + else + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "ftruncate file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + } + + close(fd); + return result; +} + +int trunk_check_and_init_file_ex(const char *filename, const int64_t file_size) +{ + struct stat file_stat; + int fd; + int result; + + result = trunk_wait_file_ready(filename, file_size, false); + if (result == 0) + { + return 0; + } + if (result == ENOENT) + { + return trunk_init_file_ex(filename, file_size); + } + if (result != ETIMEDOUT) + { + return result; + } + + if (stat(filename, &file_stat) != 0) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "stat file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + return result; + } + + logWarning("file: "__FILE__", line: %d, " \ + "file: %s, file size: "INT64_PRINTF_FORMAT \ + " < "INT64_PRINTF_FORMAT", should be resize", \ + __LINE__, filename, (int64_t)file_stat.st_size, file_size); + + fd = open(filename, O_WRONLY, 0644); + if (fd < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + return result; + } + + if (ftruncate(fd, file_size) == 0) + { + result = 0; + } + else + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "ftruncate file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, \ + result, STRERROR(result)); + } + + close(fd); + return result; +} + +bool trunk_check_size(const int64_t file_size) +{ + return file_size <= g_slot_max_size; +} + +int trunk_file_delete(const char *trunk_filename, \ + const FDFSTrunkFullInfo *pTrunkInfo) +{ + char pack_buff[FDFS_TRUNK_FILE_HEADER_SIZE]; + char buff[64 * 1024]; + int fd; + int write_bytes; + int result; + int remain_bytes; + FDFSTrunkHeader trunkHeader; + + fd = open(trunk_filename, O_WRONLY); + if (fd < 0) + { + return errno != 0 ? errno : EIO; + } + + if (lseek(fd, pTrunkInfo->file.offset, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + close(fd); + return result; + } + + memset(&trunkHeader, 0, sizeof(trunkHeader)); + trunkHeader.alloc_size = pTrunkInfo->file.size; + trunkHeader.file_type = FDFS_TRUNK_FILE_TYPE_NONE; + trunk_pack_header(&trunkHeader, pack_buff); + + write_bytes = write(fd, pack_buff, FDFS_TRUNK_FILE_HEADER_SIZE); + if (write_bytes != FDFS_TRUNK_FILE_HEADER_SIZE) + { + result = errno != 0 ? errno : EIO; + close(fd); + return result; + } + + memset(buff, 0, sizeof(buff)); + result = 0; + remain_bytes = pTrunkInfo->file.size - FDFS_TRUNK_FILE_HEADER_SIZE; + while (remain_bytes > 0) + { + write_bytes = remain_bytes > sizeof(buff) ? \ + sizeof(buff) : remain_bytes; + if (write(fd, buff, write_bytes) != write_bytes) + { + result = errno != 0 ? errno : EIO; + break; + } + + remain_bytes -= write_bytes; + } + + close(fd); + return result; +} + +int trunk_create_trunk_file_advance(void *args) +{ + int64_t total_mb_sum; + int64_t free_mb_sum; + int64_t alloc_space; + FDFSTrunkNode *pTrunkNode; + int result; + int i; + int file_count; + + if (!g_trunk_create_file_advance) + { + logError("file: "__FILE__", line: %d, " \ + "do not need create trunk file advancely!", __LINE__); + return EINVAL; + } + + if (!g_if_trunker_self) + { + logError("file: "__FILE__", line: %d, " \ + "I am not trunk server!", __LINE__); + return ENOENT; + } + + alloc_space = g_trunk_create_file_space_threshold - \ + g_trunk_total_free_space; + if (alloc_space <= 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "do not need create trunk file!", __LINE__); + return 0; + } + + total_mb_sum = 0; + free_mb_sum = 0; + for (i=0; i +#include +#include +#include +#include +#include "common_define.h" +#include "fdfs_global.h" +#include "fast_mblock.h" +#include "trunk_shared.h" +#include "fdfs_shared_func.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int g_slot_min_size; //slot min size, such as 256 bytes +extern int g_slot_max_size; //slot max size +extern int g_trunk_file_size; //the trunk file size, such as 64MB +extern int g_store_path_mode; //store which path mode, fetch from tracker +extern FDFSStorageReservedSpace g_storage_reserved_space; //fetch from tracker +extern int g_avg_storage_reserved_mb; //calc by above var: g_storage_reserved_mb +extern int g_store_path_index; //store to which path +extern int g_current_trunk_file_id; //current trunk file id +extern TimeInfo g_trunk_create_file_time_base; +extern int g_trunk_create_file_interval; +extern int g_trunk_compress_binlog_min_interval; +extern ConnectionInfo g_trunk_server; //the trunk server +extern bool g_if_use_trunk_file; //if use trunk file +extern bool g_trunk_create_file_advance; +extern bool g_trunk_init_check_occupying; +extern bool g_trunk_init_reload_from_binlog; +extern bool g_if_trunker_self; //if am i trunk server +extern int64_t g_trunk_create_file_space_threshold; +extern int64_t g_trunk_total_free_space; //trunk total free space in bytes +extern time_t g_trunk_last_compress_time; + +typedef struct tagFDFSTrunkNode { + FDFSTrunkFullInfo trunk; //trunk info + struct fast_mblock_node *pMblockNode; //for free + struct tagFDFSTrunkNode *next; +} FDFSTrunkNode; + +typedef struct { + int size; + FDFSTrunkNode *head; + struct fast_mblock_node *pMblockNode; //for free +} FDFSTrunkSlot; + +int storage_trunk_init(); +int storage_trunk_destroy_ex(const bool bNeedSleep); + +#define storage_trunk_destroy() storage_trunk_destroy_ex(false) + +int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult); +int trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, const int status); + +int trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo, \ + const bool bWriteBinLog); + +bool trunk_check_size(const int64_t file_size); + +#define trunk_init_file(filename) \ + trunk_init_file_ex(filename, g_trunk_file_size) + +#define trunk_check_and_init_file(filename) \ + trunk_check_and_init_file_ex(filename, g_trunk_file_size) + +int trunk_init_file_ex(const char *filename, const int64_t file_size); + +int trunk_check_and_init_file_ex(const char *filename, const int64_t file_size); + +int trunk_file_delete(const char *trunk_filename, \ + const FDFSTrunkFullInfo *pTrunkInfo); + +int trunk_create_trunk_file_advance(void *args); + +int storage_delete_trunk_data_file(); + +char *storage_trunk_get_data_filename(char *full_filename); + +#define storage_check_reserved_space(pGroup) \ + fdfs_check_reserved_space(pGroup, &g_storage_reserved_space) + +#define storage_check_reserved_space_trunk(pGroup) \ + fdfs_check_reserved_space_trunk(pGroup, &g_storage_reserved_space) + +#define storage_check_reserved_space_path(total_mb, free_mb, avg_mb) \ + fdfs_check_reserved_space_path(total_mb, free_mb, avg_mb, \ + &g_storage_reserved_space) + +#define storage_get_storage_reserved_space_mb(total_mb) \ + fdfs_get_storage_reserved_space_mb(total_mb, &g_storage_reserved_space) + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/trunk_mgr/trunk_shared.c b/storage/trunk_mgr/trunk_shared.c new file mode 100644 index 0000000..e77dacc --- /dev/null +++ b/storage/trunk_mgr/trunk_shared.c @@ -0,0 +1,748 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_shared.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "shared_func.h" +#include "trunk_shared.h" +#include "tracker_proto.h" + +FDFSStorePaths g_fdfs_store_paths = {0, NULL}; +struct base64_context g_fdfs_base64_context; + +void trunk_shared_init() +{ + base64_init_ex(&g_fdfs_base64_context, 0, '-', '_', '.'); +} + +char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \ + const char *szSectionName, const bool bUseBasePath, \ + int *path_count, int *err_no) +{ + char item_name[64]; + char **store_paths; + char *pPath; + int i; + + *path_count = iniGetIntValue(szSectionName, "store_path_count", + pItemContext, 1); + if (*path_count <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "store_path_count: %d is invalid!", \ + __LINE__, *path_count); + *err_no = EINVAL; + return NULL; + } + + store_paths = (char **)malloc(sizeof(char *) * (*path_count)); + if (store_paths == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(char *) * (*path_count), \ + errno, STRERROR(errno)); + *err_no = errno != 0 ? errno : ENOMEM; + return NULL; + } + memset(store_paths, 0, sizeof(char *) * (*path_count)); + + pPath = iniGetStrValue(szSectionName, "store_path0", pItemContext); + if (pPath == NULL) + { + if (!bUseBasePath) + { + logError("file: "__FILE__", line: %d, " \ + "conf file must have item " \ + "\"store_path0\"!", __LINE__); + *err_no = ENOENT; + free(store_paths); + return NULL; + } + + pPath = g_fdfs_base_path; + } + store_paths[0] = strdup(pPath); + if (store_paths[0] == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)strlen(pPath), \ + errno, STRERROR(errno)); + *err_no = errno != 0 ? errno : ENOMEM; + free(store_paths); + return NULL; + } + + *err_no = 0; + for (i=1; i<*path_count; i++) + { + sprintf(item_name, "store_path%d", i); + pPath = iniGetStrValue(szSectionName, item_name, \ + pItemContext); + if (pPath == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file must have item \"%s\"!", \ + __LINE__, item_name); + *err_no = ENOENT; + break; + } + + chopPath(pPath); + if (!fileExists(pPath)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" can't be accessed, " \ + "errno: %d, error info: %s", __LINE__, \ + pPath, errno, STRERROR(errno)); + *err_no = errno != 0 ? errno : ENOENT; + break; + } + if (!isDir(pPath)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" is not a directory!", \ + __LINE__, pPath); + *err_no = ENOTDIR; + break; + } + + store_paths[i] = strdup(pPath); + if (store_paths[i] == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", __LINE__, \ + (int)strlen(pPath), errno, STRERROR(errno)); + *err_no = errno != 0 ? errno : ENOMEM; + break; + } + } + + if (*err_no != 0) + { + for (i=0; i<*path_count; i++) + { + if (store_paths[i] != NULL) + { + free(store_paths[i]); + } + } + free(store_paths); + return NULL; + } + + return store_paths; +} + +int storage_load_paths_from_conf_file(IniContext *pItemContext) +{ + char *pPath; + int result; + + pPath = iniGetStrValue(NULL, "base_path", pItemContext); + if (pPath == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file must have item \"base_path\"!", __LINE__); + return ENOENT; + } + + snprintf(g_fdfs_base_path, sizeof(g_fdfs_base_path), "%s", pPath); + chopPath(g_fdfs_base_path); + if (!fileExists(g_fdfs_base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" can't be accessed, error info: %s", \ + __LINE__, STRERROR(errno), g_fdfs_base_path); + return errno != 0 ? errno : ENOENT; + } + if (!isDir(g_fdfs_base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" is not a directory!", \ + __LINE__, g_fdfs_base_path); + return ENOTDIR; + } + + g_fdfs_store_paths.paths = storage_load_paths_from_conf_file_ex( \ + pItemContext, NULL, true, &g_fdfs_store_paths.count, &result); + + return result; +} + +#define SPLIT_FILENAME_BODY(logic_filename, filename_len, true_filename, \ + store_path_index, check_path_index) \ + do \ + { \ + char buff[3]; \ + char *pEnd; \ + \ + if (*filename_len <= FDFS_LOGIC_FILE_PATH_LEN) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "filename_len: %d is invalid, <= %d", \ + __LINE__, *filename_len, FDFS_LOGIC_FILE_PATH_LEN); \ + return EINVAL; \ + } \ + \ + if (*logic_filename != FDFS_STORAGE_STORE_PATH_PREFIX_CHAR) \ + { /* version < V1.12 */ \ + store_path_index = 0; \ + memcpy(true_filename, logic_filename, (*filename_len)+1); \ + break; \ + } \ + \ + if (*(logic_filename + 3) != '/') \ + { \ + logError("file: "__FILE__", line: %d, " \ + "filename: %s is invalid", \ + __LINE__, logic_filename); \ + return EINVAL; \ + } \ + \ + *buff = *(logic_filename+1); \ + *(buff+1) = *(logic_filename+2); \ + *(buff+2) = '\0'; \ + \ + pEnd = NULL; \ + store_path_index = strtol(buff, &pEnd, 16); \ + if (pEnd != NULL && *pEnd != '\0') \ + { \ + logError("file: "__FILE__", line: %d, " \ + "filename: %s is invalid", \ + __LINE__, logic_filename); \ + return EINVAL; \ + } \ + \ + if (check_path_index && (store_path_index < 0 || \ + store_path_index >= g_fdfs_store_paths.count)) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "filename: %s is invalid, " \ + "invalid store path index: %d", \ + __LINE__, logic_filename, store_path_index); \ + return EINVAL; \ + } \ + \ + *filename_len -= 4; \ + memcpy(true_filename, logic_filename + 4, (*filename_len) + 1); \ + \ + } while (0) + + +int storage_split_filename(const char *logic_filename, \ + int *filename_len, char *true_filename, char **ppStorePath) +{ + int store_path_index; + + SPLIT_FILENAME_BODY(logic_filename, filename_len, true_filename, \ + store_path_index, true); + + *ppStorePath = g_fdfs_store_paths.paths[store_path_index]; + + return 0; +} + +int storage_split_filename_ex(const char *logic_filename, \ + int *filename_len, char *true_filename, int *store_path_index) +{ + SPLIT_FILENAME_BODY(logic_filename, \ + filename_len, true_filename, *store_path_index, true); + + return 0; +} + +int storage_split_filename_no_check(const char *logic_filename, \ + int *filename_len, char *true_filename, int *store_path_index) +{ + SPLIT_FILENAME_BODY(logic_filename, \ + filename_len, true_filename, *store_path_index, false); + + return 0; +} + +char *trunk_info_dump(const FDFSTrunkFullInfo *pTrunkInfo, char *buff, \ + const int buff_size) +{ + snprintf(buff, buff_size, \ + "store_path_index=%d, " \ + "sub_path_high=%d, " \ + "sub_path_low=%d, " \ + "id=%d, offset=%d, size=%d, status=%d", \ + pTrunkInfo->path.store_path_index, \ + pTrunkInfo->path.sub_path_high, \ + pTrunkInfo->path.sub_path_low, \ + pTrunkInfo->file.id, pTrunkInfo->file.offset, pTrunkInfo->file.size, \ + pTrunkInfo->status); + + return buff; +} + +char *trunk_header_dump(const FDFSTrunkHeader *pTrunkHeader, char *buff, \ + const int buff_size) +{ + snprintf(buff, buff_size, \ + "file_type=%d, " \ + "alloc_size=%d, " \ + "file_size=%d, " \ + "crc32=%d, " \ + "mtime=%d, " \ + "ext_name(%d)=%s", \ + pTrunkHeader->file_type, pTrunkHeader->alloc_size, \ + pTrunkHeader->file_size, pTrunkHeader->crc32, \ + pTrunkHeader->mtime, \ + (int)strlen(pTrunkHeader->formatted_ext_name), \ + pTrunkHeader->formatted_ext_name); + + return buff; +} + +char *trunk_get_full_filename_ex(const FDFSStorePaths *pStorePaths, \ + const FDFSTrunkFullInfo *pTrunkInfo, \ + char *full_filename, const int buff_size) +{ + char short_filename[64]; + char *pStorePath; + + pStorePath = pStorePaths->paths[pTrunkInfo->path.store_path_index]; + TRUNK_GET_FILENAME(pTrunkInfo->file.id, short_filename); + + snprintf(full_filename, buff_size, \ + "%s/data/"FDFS_STORAGE_DATA_DIR_FORMAT"/" \ + FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \ + pStorePath, pTrunkInfo->path.sub_path_high, \ + pTrunkInfo->path.sub_path_low, short_filename); + + return full_filename; +} + +void trunk_pack_header(const FDFSTrunkHeader *pTrunkHeader, char *buff) +{ + *(buff + FDFS_TRUNK_FILE_FILE_TYPE_OFFSET) = pTrunkHeader->file_type; + int2buff(pTrunkHeader->alloc_size, \ + buff + FDFS_TRUNK_FILE_ALLOC_SIZE_OFFSET); + int2buff(pTrunkHeader->file_size, \ + buff + FDFS_TRUNK_FILE_FILE_SIZE_OFFSET); + int2buff(pTrunkHeader->crc32, \ + buff + FDFS_TRUNK_FILE_FILE_CRC32_OFFSET); + int2buff(pTrunkHeader->mtime, \ + buff + FDFS_TRUNK_FILE_FILE_MTIME_OFFSET); + memcpy(buff + FDFS_TRUNK_FILE_FILE_EXT_NAME_OFFSET, \ + pTrunkHeader->formatted_ext_name, \ + FDFS_FILE_EXT_NAME_MAX_LEN + 1); +} + +void trunk_unpack_header(const char *buff, FDFSTrunkHeader *pTrunkHeader) +{ + pTrunkHeader->file_type = *(buff + FDFS_TRUNK_FILE_FILE_TYPE_OFFSET); + pTrunkHeader->alloc_size = buff2int( + buff + FDFS_TRUNK_FILE_ALLOC_SIZE_OFFSET); + pTrunkHeader->file_size = buff2int( + buff + FDFS_TRUNK_FILE_FILE_SIZE_OFFSET); + pTrunkHeader->crc32 = buff2int( + buff + FDFS_TRUNK_FILE_FILE_CRC32_OFFSET); + pTrunkHeader->mtime = buff2int( + buff + FDFS_TRUNK_FILE_FILE_MTIME_OFFSET); + memcpy(pTrunkHeader->formatted_ext_name, buff + \ + FDFS_TRUNK_FILE_FILE_EXT_NAME_OFFSET, \ + FDFS_FILE_EXT_NAME_MAX_LEN + 1); + *(pTrunkHeader->formatted_ext_name+FDFS_FILE_EXT_NAME_MAX_LEN+1)='\0'; +} + +void trunk_file_info_encode(const FDFSTrunkFileInfo *pTrunkFile, char *str) +{ + char buff[sizeof(int) * 3]; + int len; + + int2buff(pTrunkFile->id, buff); + int2buff(pTrunkFile->offset, buff + sizeof(int)); + int2buff(pTrunkFile->size, buff + sizeof(int) * 2); + base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(buff), \ + str, &len, false); +} + +void trunk_file_info_decode(const char *str, FDFSTrunkFileInfo *pTrunkFile) +{ + char buff[FDFS_TRUNK_FILE_INFO_LEN]; + int len; + + base64_decode_auto(&g_fdfs_base64_context, str, FDFS_TRUNK_FILE_INFO_LEN, \ + buff, &len); + + pTrunkFile->id = buff2int(buff); + pTrunkFile->offset = buff2int(buff + sizeof(int)); + pTrunkFile->size = buff2int(buff + sizeof(int) * 2); +} + +int trunk_file_get_content_ex(const FDFSStorePaths *pStorePaths, \ + const FDFSTrunkFullInfo *pTrunkInfo, const int file_size, \ + int *pfd, char *buff, const int buff_size) +{ + char full_filename[MAX_PATH_SIZE]; + int fd; + int result; + int read_bytes; + + if (file_size > buff_size) + { + return ENOSPC; + } + + if (pfd != NULL) + { + fd = *pfd; + } + else + { + trunk_get_full_filename_ex(pStorePaths, pTrunkInfo, \ + full_filename, sizeof(full_filename)); + fd = open(full_filename, O_RDONLY); + if (fd < 0) + { + return errno != 0 ? errno : EIO; + } + + if (lseek(fd, pTrunkInfo->file.offset + \ + FDFS_TRUNK_FILE_HEADER_SIZE, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + close(fd); + return result; + } + } + + read_bytes = read(fd, buff, file_size); + if (read_bytes == file_size) + { + result = 0; + } + else + { + result = errno != 0 ? errno : EINVAL; + } + + if (pfd == NULL) + { + close(fd); + } + + return result; +} + +int trunk_file_stat_func_ex(const FDFSStorePaths *pStorePaths, \ + const int store_path_index, const char *true_filename, \ + const int filename_len, const int stat_func, \ + struct stat *pStat, FDFSTrunkFullInfo *pTrunkInfo, \ + FDFSTrunkHeader *pTrunkHeader, int *pfd) +{ + int result; + int src_store_path_index; + int src_filename_len; + char src_filename[128]; + char src_true_filename[128]; + + result = trunk_file_do_lstat_func_ex(pStorePaths, store_path_index, \ + true_filename, filename_len, stat_func, \ + pStat, pTrunkInfo, pTrunkHeader, pfd); + if (result != 0) + { + return result; + } + + if (!(stat_func == FDFS_STAT_FUNC_STAT && IS_TRUNK_FILE_BY_ID( \ + (*pTrunkInfo)) && S_ISLNK(pStat->st_mode))) + { + return 0; + } + + do + { + result = trunk_file_get_content_ex(pStorePaths, pTrunkInfo, \ + pStat->st_size, pfd, src_filename, \ + sizeof(src_filename) - 1); + if (result != 0) + { + break; + } + + src_filename_len = pStat->st_size; + *(src_filename + src_filename_len) = '\0'; + if ((result=storage_split_filename_no_check(src_filename, \ + &src_filename_len, src_true_filename, \ + &src_store_path_index)) != 0) + { + break; + } + if (src_store_path_index < 0 || \ + src_store_path_index >= pStorePaths->count) + { + logError("file: "__FILE__", line: %d, " \ + "filename: %s is invalid, " \ + "invalid store path index: %d, " \ + "which < 0 or >= %d", __LINE__, \ + src_filename, src_store_path_index, \ + pStorePaths->count); + result = EINVAL; + break; + } + + if (pfd != NULL) + { + close(*pfd); + *pfd = -1; + } + + result = trunk_file_do_lstat_func_ex(pStorePaths, \ + src_store_path_index, src_true_filename, \ + src_filename_len, stat_func, pStat, \ + pTrunkInfo, pTrunkHeader, pfd); + } while (0); + + if (result != 0 && pfd != NULL && *pfd >= 0) + { + close(*pfd); + *pfd = -1; + } + + return result; +} + +int trunk_file_do_lstat_func_ex(const FDFSStorePaths *pStorePaths, \ + const int store_path_index, const char *true_filename, \ + const int filename_len, const int stat_func, \ + struct stat *pStat, FDFSTrunkFullInfo *pTrunkInfo, \ + FDFSTrunkHeader *pTrunkHeader, int *pfd) +{ + char full_filename[MAX_PATH_SIZE]; + char buff[128]; + char pack_buff[FDFS_TRUNK_FILE_HEADER_SIZE]; + int64_t file_size; + int buff_len; + int fd; + int read_bytes; + int result; + + pTrunkInfo->file.id = 0; + if (filename_len != FDFS_TRUNK_FILENAME_LENGTH) //not trunk file + { + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + pStorePaths->paths[store_path_index], true_filename); + + if (stat_func == FDFS_STAT_FUNC_STAT) + { + result = stat(full_filename, pStat); + } + else + { + result = lstat(full_filename, pStat); + } + if (result == 0) + { + return 0; + } + else + { + return errno != 0 ? errno : ENOENT; + } + } + + memset(buff, 0, sizeof(buff)); + base64_decode_auto(&g_fdfs_base64_context, (char *)true_filename + \ + FDFS_TRUE_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + buff, &buff_len); + + file_size = buff2long(buff + sizeof(int) * 2); + if (!IS_TRUNK_FILE(file_size)) //slave file + { + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + pStorePaths->paths[store_path_index], true_filename); + + if (stat_func == FDFS_STAT_FUNC_STAT) + { + result = stat(full_filename, pStat); + } + else + { + result = lstat(full_filename, pStat); + } + if (result == 0) + { + return 0; + } + else + { + return errno != 0 ? errno : ENOENT; + } + } + + trunk_file_info_decode(true_filename + FDFS_TRUE_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH, &pTrunkInfo->file); + + pTrunkHeader->file_size = FDFS_TRUNK_FILE_TRUE_SIZE(file_size); + pTrunkHeader->mtime = buff2int(buff + sizeof(int)); + pTrunkHeader->crc32 = buff2int(buff + sizeof(int) * 4); + memcpy(pTrunkHeader->formatted_ext_name, true_filename + \ + (filename_len - (FDFS_FILE_EXT_NAME_MAX_LEN + 1)), \ + FDFS_FILE_EXT_NAME_MAX_LEN + 2); //include tailing '\0' + pTrunkHeader->alloc_size = pTrunkInfo->file.size; + + pTrunkInfo->path.store_path_index = store_path_index; + pTrunkInfo->path.sub_path_high = strtol(true_filename, NULL, 16); + pTrunkInfo->path.sub_path_low = strtol(true_filename + 3, NULL, 16); + + trunk_get_full_filename_ex(pStorePaths, pTrunkInfo, full_filename, \ + sizeof(full_filename)); + fd = open(full_filename, O_RDONLY); + if (fd < 0) + { + return errno != 0 ? errno : EIO; + } + + if (lseek(fd, pTrunkInfo->file.offset, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + close(fd); + return result; + } + + read_bytes = read(fd, buff, FDFS_TRUNK_FILE_HEADER_SIZE); + if (read_bytes == FDFS_TRUNK_FILE_HEADER_SIZE) + { + result = 0; + } + else + { + result = errno; + close(fd); + return result != 0 ? result : EINVAL; + } + + memset(pStat, 0, sizeof(struct stat)); + pTrunkHeader->file_type = *(buff + FDFS_TRUNK_FILE_FILE_TYPE_OFFSET); + if (pTrunkHeader->file_type == FDFS_TRUNK_FILE_TYPE_REGULAR) + { + pStat->st_mode = S_IFREG; + } + else if (pTrunkHeader->file_type == FDFS_TRUNK_FILE_TYPE_LINK) + { + pStat->st_mode = S_IFLNK; + } + else if (pTrunkHeader->file_type == FDFS_TRUNK_FILE_TYPE_NONE) + { + close(fd); + return ENOENT; + } + else + { + close(fd); + logError("file: "__FILE__", line: %d, " \ + "Invalid file type: %d", __LINE__, \ + pTrunkHeader->file_type); + return ENOENT; + } + + trunk_pack_header(pTrunkHeader, pack_buff); + + /* + { + char temp[265]; + char szHexBuff[2 * FDFS_TRUNK_FILE_HEADER_SIZE + 1]; + FDFSTrunkHeader trueTrunkHeader; + + fprintf(stderr, "file: "__FILE__", line: %d, true buff=%s\n", __LINE__, \ + bin2hex(buff+1, FDFS_TRUNK_FILE_HEADER_SIZE - 1, szHexBuff)); + trunk_unpack_header(buff, &trueTrunkHeader); + fprintf(stderr, "file: "__FILE__", line: %d, true fields=%s\n", __LINE__, \ + trunk_header_dump(&trueTrunkHeader, full_filename, sizeof(full_filename))); + + fprintf(stderr, "file: "__FILE__", line: %d, my buff=%s\n", __LINE__, \ + bin2hex(pack_buff+1, FDFS_TRUNK_FILE_HEADER_SIZE - 1, szHexBuff)); + fprintf(stderr, "file: "__FILE__", line: %d, my trunk=%s, my fields=%s\n", __LINE__, \ + trunk_info_dump(pTrunkInfo, temp, sizeof(temp)), \ + trunk_header_dump(pTrunkHeader, full_filename, sizeof(full_filename))); + } + */ + + if (memcmp(pack_buff, buff, FDFS_TRUNK_FILE_HEADER_SIZE) != 0) + { + close(fd); + return ENOENT; + } + + pStat->st_size = pTrunkHeader->file_size; + pStat->st_mtime = pTrunkHeader->mtime; + + if (pfd != NULL) + { + *pfd = fd; + } + else + { + close(fd); + } + + return 0; +} + +bool fdfs_is_trunk_file(const char *remote_filename, const int filename_len) +{ + int buff_len; + char buff[64]; + int64_t file_size; + + if (filename_len != FDFS_TRUNK_LOGIC_FILENAME_LENGTH) //not trunk file + { + return false; + } + + memset(buff, 0, sizeof(buff)); + base64_decode_auto(&g_fdfs_base64_context, (char *)remote_filename + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + buff, &buff_len); + + file_size = buff2long(buff + sizeof(int) * 2); + return IS_TRUNK_FILE(file_size); +} + +int fdfs_decode_trunk_info(const int store_path_index, \ + const char *true_filename, const int filename_len, \ + FDFSTrunkFullInfo *pTrunkInfo) +{ + if (filename_len != FDFS_TRUNK_FILENAME_LENGTH) //not trunk file + { + logWarning("file: "__FILE__", line: %d, " \ + "trunk filename length: %d != %d, filename: %s", \ + __LINE__, filename_len, FDFS_TRUNK_FILENAME_LENGTH, \ + true_filename); + return EINVAL; + } + + pTrunkInfo->path.store_path_index = store_path_index; + pTrunkInfo->path.sub_path_high = strtol(true_filename, NULL, 16); + pTrunkInfo->path.sub_path_low = strtol(true_filename + 3, NULL, 16); + trunk_file_info_decode(true_filename + FDFS_TRUE_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH, &pTrunkInfo->file); + return 0; +} + diff --git a/storage/trunk_mgr/trunk_shared.h b/storage/trunk_mgr/trunk_shared.h new file mode 100644 index 0000000..7f74d8c --- /dev/null +++ b/storage/trunk_mgr/trunk_shared.h @@ -0,0 +1,189 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_shared.h + +#ifndef _TRUNK_SHARED_H_ +#define _TRUNK_SHARED_H_ + +#include +#include +#include +#include +#include "base64.h" +#include "common_define.h" +#include "ini_file_reader.h" +#include "fdfs_global.h" +#include "tracker_types.h" + +#define FDFS_TRUNK_STATUS_FREE 0 +#define FDFS_TRUNK_STATUS_HOLD 1 + +#define FDFS_TRUNK_FILE_TYPE_NONE '\0' +#define FDFS_TRUNK_FILE_TYPE_REGULAR 'F' +#define FDFS_TRUNK_FILE_TYPE_LINK 'L' + +#define FDFS_STAT_FUNC_STAT 0 +#define FDFS_STAT_FUNC_LSTAT 1 + +#define FDFS_TRUNK_FILE_FILE_TYPE_OFFSET 0 +#define FDFS_TRUNK_FILE_ALLOC_SIZE_OFFSET 1 +#define FDFS_TRUNK_FILE_FILE_SIZE_OFFSET 5 +#define FDFS_TRUNK_FILE_FILE_CRC32_OFFSET 9 +#define FDFS_TRUNK_FILE_FILE_MTIME_OFFSET 13 +#define FDFS_TRUNK_FILE_FILE_EXT_NAME_OFFSET 17 +#define FDFS_TRUNK_FILE_HEADER_SIZE (17 + FDFS_FILE_EXT_NAME_MAX_LEN + 1) + +#define TRUNK_CALC_SIZE(file_size) (FDFS_TRUNK_FILE_HEADER_SIZE + file_size) +#define TRUNK_FILE_START_OFFSET(trunkInfo) \ + (FDFS_TRUNK_FILE_HEADER_SIZE + trunkInfo.file.offset) + +#define IS_TRUNK_FILE_BY_ID(trunkInfo) (trunkInfo.file.id > 0) + +#define TRUNK_GET_FILENAME(file_id, filename) \ + sprintf(filename, "%06d", file_id) + +#ifdef __cplusplus +extern "C" { +#endif + +extern FDFSStorePaths g_fdfs_store_paths; //file store paths +extern struct base64_context g_fdfs_base64_context; //base64 context + +typedef int (*stat_func)(const char *filename, struct stat *buf); + +typedef struct tagFDFSTrunkHeader { + char file_type; + char formatted_ext_name[FDFS_FILE_EXT_NAME_MAX_LEN + 2]; + int alloc_size; + int file_size; + int crc32; + int mtime; +} FDFSTrunkHeader; + +typedef struct tagFDFSTrunkPathInfo { + unsigned char store_path_index; //store which path as Mxx + unsigned char sub_path_high; //high sub dir index, front part of HH/HH + unsigned char sub_path_low; //low sub dir index, tail part of HH/HH +} FDFSTrunkPathInfo; + +typedef struct tagFDFSTrunkFileInfo { + int id; //trunk file id + int offset; //file offset + int size; //space size +} FDFSTrunkFileInfo; + +typedef struct tagFDFSTrunkFullInfo { + char status; //normal or hold + FDFSTrunkPathInfo path; + FDFSTrunkFileInfo file; +} FDFSTrunkFullInfo; + +char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \ + const char *szSectionName, const bool bUseBasePath, \ + int *path_count, int *err_no); +int storage_load_paths_from_conf_file(IniContext *pItemContext); +void trunk_shared_init(); + +int storage_split_filename(const char *logic_filename, \ + int *filename_len, char *true_filename, char **ppStorePath); +int storage_split_filename_ex(const char *logic_filename, \ + int *filename_len, char *true_filename, int *store_path_index); +int storage_split_filename_no_check(const char *logic_filename, \ + int *filename_len, char *true_filename, int *store_path_index); + +void trunk_file_info_encode(const FDFSTrunkFileInfo *pTrunkFile, char *str); +void trunk_file_info_decode(const char *str, FDFSTrunkFileInfo *pTrunkFile); + +char *trunk_info_dump(const FDFSTrunkFullInfo *pTrunkInfo, char *buff, \ + const int buff_size); + +char *trunk_header_dump(const FDFSTrunkHeader *pTrunkHeader, char *buff, \ + const int buff_size); + +#define trunk_get_full_filename(pTrunkInfo, full_filename, buff_size) \ + trunk_get_full_filename_ex(&g_fdfs_store_paths, pTrunkInfo, \ + full_filename, buff_size) + +char *trunk_get_full_filename_ex(const FDFSStorePaths *pStorePaths, \ + const FDFSTrunkFullInfo *pTrunkInfo, \ + char *full_filename, const int buff_size); + +void trunk_pack_header(const FDFSTrunkHeader *pTrunkHeader, char *buff); +void trunk_unpack_header(const char *buff, FDFSTrunkHeader *pTrunkHeader); + +#define trunk_file_get_content(pTrunkInfo, file_size, pfd, buff, buff_size) \ + trunk_file_get_content_ex(&g_fdfs_store_paths, pTrunkInfo, \ + file_size, pfd, buff, buff_size) + +int trunk_file_get_content_ex(const FDFSStorePaths *pStorePaths, \ + const FDFSTrunkFullInfo *pTrunkInfo, const int file_size, \ + int *pfd, char *buff, const int buff_size); + +#define trunk_file_do_lstat_func(store_path_index, true_filename, \ + filename_len, stat_func, pStat, pTrunkInfo, pTrunkHeader, pfd) \ + trunk_file_do_lstat_func_ex(&g_fdfs_store_paths, store_path_index, \ + true_filename, filename_len, stat_func, pStat, pTrunkInfo, \ + pTrunkHeader, pfd) + +#define trunk_file_stat_func(store_path_index, true_filename, filename_len, \ + stat_func, pStat, pTrunkInfo, pTrunkHeader, pfd) \ + trunk_file_stat_func_ex(&g_fdfs_store_paths, store_path_index, \ + true_filename, filename_len, stat_func, pStat, pTrunkInfo, \ + pTrunkHeader, pfd) + +#define trunk_file_stat(store_path_index, true_filename, filename_len, \ + pStat, pTrunkInfo, pTrunkHeader) \ + trunk_file_stat_func(store_path_index, true_filename, filename_len, \ + FDFS_STAT_FUNC_STAT, pStat, pTrunkInfo, pTrunkHeader, NULL) + +#define trunk_file_lstat(store_path_index, true_filename, filename_len, \ + pStat, pTrunkInfo, pTrunkHeader) \ + trunk_file_do_lstat_func(store_path_index, true_filename, filename_len, \ + FDFS_STAT_FUNC_LSTAT, pStat, pTrunkInfo, pTrunkHeader, NULL) + +#define trunk_file_lstat_ex(store_path_index, true_filename, filename_len, \ + pStat, pTrunkInfo, pTrunkHeader, pfd) \ + trunk_file_do_lstat_func(store_path_index, true_filename, filename_len, \ + FDFS_STAT_FUNC_LSTAT, pStat, pTrunkInfo, pTrunkHeader, pfd) + +#define trunk_file_stat_ex(store_path_index, true_filename, filename_len, \ + pStat, pTrunkInfo, pTrunkHeader, pfd) \ + trunk_file_stat_func(store_path_index, true_filename, filename_len, \ + FDFS_STAT_FUNC_STAT, pStat, pTrunkInfo, pTrunkHeader, pfd) + +#define trunk_file_stat_ex1(pStorePaths, store_path_index, true_filename, \ + filename_len, pStat, pTrunkInfo, pTrunkHeader, pfd) \ + trunk_file_stat_func_ex(pStorePaths, store_path_index, true_filename, \ + filename_len, FDFS_STAT_FUNC_STAT, pStat, pTrunkInfo, \ + pTrunkHeader, pfd) + +int trunk_file_stat_func_ex(const FDFSStorePaths *pStorePaths, \ + const int store_path_index, const char *true_filename, \ + const int filename_len, const int stat_func, \ + struct stat *pStat, FDFSTrunkFullInfo *pTrunkInfo, \ + FDFSTrunkHeader *pTrunkHeader, int *pfd); + +int trunk_file_do_lstat_func_ex(const FDFSStorePaths *pStorePaths, \ + const int store_path_index, const char *true_filename, \ + const int filename_len, const int stat_func, \ + struct stat *pStat, FDFSTrunkFullInfo *pTrunkInfo, \ + FDFSTrunkHeader *pTrunkHeader, int *pfd); + +bool fdfs_is_trunk_file(const char *remote_filename, const int filename_len); + +int fdfs_decode_trunk_info(const int store_path_index, \ + const char *true_filename, const int filename_len, \ + FDFSTrunkFullInfo *pTrunkInfo); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/storage/trunk_mgr/trunk_sync.c b/storage/trunk_mgr/trunk_sync.c new file mode 100644 index 0000000..8c4e286 --- /dev/null +++ b/storage/trunk_mgr/trunk_sync.c @@ -0,0 +1,1811 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_sync.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "ini_file_reader.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "storage_global.h" +#include "storage_func.h" +#include "storage_ip_changed_dealer.h" +#include "tracker_client_thread.h" +#include "storage_client.h" +#include "trunk_sync.h" + +#define TRUNK_SYNC_BINLOG_FILENAME "binlog" +#define TRUNK_SYNC_BINLOG_ROLLBACK_EXT ".rollback" +#define TRUNK_SYNC_MARK_FILE_EXT ".mark" +#define TRUNK_DIR_NAME "trunk" +#define MARK_ITEM_BINLOG_FILE_OFFSET "binlog_offset" + +static int trunk_binlog_fd = -1; + +int g_trunk_sync_thread_count = 0; +static pthread_mutex_t trunk_sync_thread_lock; +static char *trunk_binlog_write_cache_buff = NULL; +static int trunk_binlog_write_cache_len = 0; +static int trunk_binlog_write_version = 1; + +/* save sync thread ids */ +static pthread_t *trunk_sync_tids = NULL; + +static int trunk_write_to_mark_file(TrunkBinLogReader *pReader); +static int trunk_binlog_fsync_ex(const bool bNeedLock, \ + const char *buff, int *length); +static int trunk_binlog_preread(TrunkBinLogReader *pReader); + +#define trunk_binlog_fsync(bNeedLock) trunk_binlog_fsync_ex(bNeedLock, \ + trunk_binlog_write_cache_buff, (&trunk_binlog_write_cache_len)) + +char *get_trunk_binlog_filename(char *full_filename) +{ + snprintf(full_filename, MAX_PATH_SIZE, \ + "%s/data/"TRUNK_DIR_NAME"/"TRUNK_SYNC_BINLOG_FILENAME, \ + g_fdfs_base_path); + return full_filename; +} + +static char *get_trunk_rollback_filename(char *full_filename) +{ + get_trunk_binlog_filename(full_filename); + if (strlen(full_filename) + sizeof(TRUNK_SYNC_BINLOG_ROLLBACK_EXT) > + MAX_PATH_SIZE) + { + return NULL; + } + strcat(full_filename, TRUNK_SYNC_BINLOG_ROLLBACK_EXT); + return full_filename; +} + +static int trunk_binlog_open_writer(const char *binlog_filename) +{ + trunk_binlog_fd = open(binlog_filename, O_WRONLY | O_CREAT | + O_APPEND, 0644); + if (trunk_binlog_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, binlog_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +static int trunk_binlog_close_writer(const bool needLock) +{ + int result; + if (trunk_binlog_write_cache_len > 0) + { + if ((result=trunk_binlog_fsync(needLock)) != 0) + { + return result; + } + } + close(trunk_binlog_fd); + trunk_binlog_fd = -1; + return 0; +} + +int trunk_sync_init() +{ + char data_path[MAX_PATH_SIZE]; + char sync_path[MAX_PATH_SIZE]; + char binlog_filename[MAX_PATH_SIZE]; + int result; + + snprintf(data_path, sizeof(data_path), "%s/data", g_fdfs_base_path); + if (!fileExists(data_path)) + { + if (mkdir(data_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + STORAGE_CHOWN(data_path, geteuid(), getegid()) + } + + snprintf(sync_path, sizeof(sync_path), \ + "%s/"TRUNK_DIR_NAME, data_path); + if (!fileExists(sync_path)) + { + if (mkdir(sync_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, sync_path, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + STORAGE_CHOWN(sync_path, geteuid(), getegid()) + } + + trunk_binlog_write_cache_buff = (char *)malloc( \ + TRUNK_BINLOG_BUFFER_SIZE); + if (trunk_binlog_write_cache_buff == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, TRUNK_BINLOG_BUFFER_SIZE, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + get_trunk_binlog_filename(binlog_filename); + if ((result=trunk_binlog_open_writer(binlog_filename)) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&trunk_sync_thread_lock)) != 0) + { + return result; + } + + STORAGE_FCHOWN(trunk_binlog_fd, binlog_filename, geteuid(), getegid()) + + return 0; +} + +int trunk_sync_destroy() +{ + if (trunk_binlog_fd >= 0) + { + trunk_binlog_fsync(true); + close(trunk_binlog_fd); + trunk_binlog_fd = -1; + } + + return 0; +} + +int kill_trunk_sync_threads() +{ + int result; + int kill_res; + + if (trunk_sync_tids == NULL) + { + return 0; + } + + if ((result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + kill_res = kill_work_threads(trunk_sync_tids, g_trunk_sync_thread_count); + + if ((result=pthread_mutex_unlock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + while (g_trunk_sync_thread_count > 0) + { + usleep(50000); + } + + return kill_res; +} + +int trunk_binlog_sync_func(void *args) +{ + if (trunk_binlog_write_cache_len > 0) + { + return trunk_binlog_fsync(true); + } + else + { + return 0; + } +} + +int trunk_binlog_truncate() +{ + int result; + + if (trunk_binlog_write_cache_len > 0) + { + if ((result=trunk_binlog_fsync(true)) != 0) + { + return result; + } + } + + if (ftruncate(trunk_binlog_fd, 0) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "call ftruncate fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int trunk_binlog_compress_apply() +{ + int result; + char binlog_filename[MAX_PATH_SIZE]; + char rollback_filename[MAX_PATH_SIZE]; + + get_trunk_binlog_filename(binlog_filename); + if (get_trunk_rollback_filename(rollback_filename) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "filename: %s is too long", + __LINE__, binlog_filename); + return ENAMETOOLONG; + } + + if (trunk_binlog_fd < 0) + { + if (access(binlog_filename, F_OK) == 0) + { + if (rename(binlog_filename, rollback_filename) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "rename %s to %s fail, " \ + "errno: %d, error info: %s", + __LINE__, binlog_filename, + rollback_filename, result, + STRERROR(result)); + return result; + } + } + else if (errno != ENOENT) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "call access %s fail, " \ + "errno: %d, error info: %s", + __LINE__, binlog_filename, + result, STRERROR(result)); + return result; + } + + return 0; + } + + if ((result=trunk_binlog_close_writer(true)) != 0) + { + return result; + } + + if (rename(binlog_filename, rollback_filename) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "rename %s to %s fail, " \ + "errno: %d, error info: %s", + __LINE__, binlog_filename, rollback_filename, + result, STRERROR(result)); + return result; + } + + if ((result=trunk_binlog_open_writer(binlog_filename)) != 0) + { + rename(rollback_filename, binlog_filename); //rollback + return result; + } + + return 0; +} + +static int trunk_binlog_open_read(const char *filename, + const bool skipFirstLine) +{ + int result; + int fd; + char buff[32]; + + fd = open(filename, O_RDONLY); + if (fd < 0) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, result, STRERROR(result)); + return -1; + } + + if (skipFirstLine) + { + if (fd_gets(fd, buff, sizeof(buff), 16) <= 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "skip first line fail!", __LINE__); + } + } + + return fd; +} + +static int trunk_binlog_merge_file(int old_fd) +{ + int result; + int tmp_fd; + int bytes; + char binlog_filename[MAX_PATH_SIZE]; + char tmp_filename[MAX_PATH_SIZE]; + char buff[64 * 1024]; + + get_trunk_binlog_filename(binlog_filename); + sprintf(tmp_filename, "%s.tmp", binlog_filename); + tmp_fd = open(tmp_filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (tmp_fd < 0) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "open file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmp_filename, result, STRERROR(result)); + return result; + } + + while ((bytes=read(old_fd, buff, sizeof(buff))) > 0) + { + if (write(tmp_fd, buff, bytes) != bytes) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmp_filename, + result, STRERROR(result)); + close(tmp_fd); + return result; + } + } + + if (access(binlog_filename, F_OK) == 0) + { + int binlog_fd; + if ((binlog_fd=trunk_binlog_open_read(binlog_filename, + false)) < 0) + { + close(tmp_fd); + return errno != 0 ? errno : EPERM; + } + + while ((bytes=read(binlog_fd, buff, sizeof(buff))) > 0) + { + if (write(tmp_fd, buff, bytes) != bytes) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmp_filename, + result, STRERROR(result)); + close(tmp_fd); + close(binlog_fd); + return result; + } + } + close(binlog_fd); + } + + if (fsync(tmp_fd) != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "sync file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmp_filename, \ + errno, STRERROR(errno)); + close(tmp_fd); + return result; + } + close(tmp_fd); + + if (rename(tmp_filename, binlog_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "rename %s to %s fail, " \ + "errno: %d, error info: %s", + __LINE__, tmp_filename, binlog_filename, + result, STRERROR(result)); + return result; + } + + return 0; +} + +int trunk_binlog_compress_commit() +{ + int result; + int data_fd; + bool need_open_binlog; + char binlog_filename[MAX_PATH_SIZE]; + char data_filename[MAX_PATH_SIZE]; + char rollback_filename[MAX_PATH_SIZE]; + + need_open_binlog = trunk_binlog_fd >= 0; + get_trunk_binlog_filename(binlog_filename); + storage_trunk_get_data_filename(data_filename); + + if ((data_fd=trunk_binlog_open_read(data_filename, true)) < 0) + { + return errno != 0 ? errno : ENOENT; + } + + if (need_open_binlog) + { + trunk_binlog_close_writer(true); + } + + result = trunk_binlog_merge_file(data_fd); + close(data_fd); + if (result != 0) + { + return result; + } + if (unlink(data_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "unlink %s fail, errno: %d, error info: %s", + __LINE__, data_filename, + result, STRERROR(result)); + return result; + } + + get_trunk_rollback_filename(rollback_filename); + if (access(rollback_filename, F_OK) == 0) + { + if (unlink(rollback_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logWarning("file: "__FILE__", line: %d, " \ + "unlink %s fail, errno: %d, error info: %s", + __LINE__, rollback_filename, + result, STRERROR(result)); + } + } + + if (need_open_binlog) + { + return trunk_binlog_open_writer(binlog_filename); + } + + return 0; +} + +int trunk_binlog_compress_rollback() +{ + int result; + int rollback_fd; + char binlog_filename[MAX_PATH_SIZE]; + char rollback_filename[MAX_PATH_SIZE]; + struct stat fs; + + get_trunk_binlog_filename(binlog_filename); + get_trunk_rollback_filename(rollback_filename); + if (trunk_binlog_fd < 0) + { + if (access(rollback_filename, F_OK) == 0) + { + if (rename(rollback_filename, binlog_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, "\ + "rename %s to %s fail, " \ + "errno: %d, error info: %s", + __LINE__, rollback_filename, + binlog_filename, result, + STRERROR(result)); + return result; + } + } + + return 0; + } + + if (stat(rollback_filename, &fs) != 0) + { + result = errno != 0 ? errno : ENOENT; + if (result == ENOENT) + { + return 0; + } + logError("file: "__FILE__", line: %d, " \ + "stat file %s fail, errno: %d, error info: %s", + __LINE__, rollback_filename, + result, STRERROR(result)); + return result; + } + + if (fs.st_size == 0) + { + unlink(rollback_filename); //delete zero file directly + return 0; + } + + if ((result=trunk_binlog_close_writer(true)) != 0) + { + return result; + } + + if ((rollback_fd=trunk_binlog_open_read(rollback_filename, + false)) < 0) + { + return errno != 0 ? errno : ENOENT; + } + + result = trunk_binlog_merge_file(rollback_fd); + close(rollback_fd); + if (result == 0) + { + if (unlink(rollback_filename) != 0) + { + result = errno != 0 ? errno : EPERM; + logWarning("file: "__FILE__", line: %d, " \ + "unlink %s fail, " \ + "errno: %d, error info: %s", + __LINE__, rollback_filename, + result, STRERROR(result)); + } + + return trunk_binlog_open_writer(binlog_filename); + } + else + { + trunk_binlog_open_writer(binlog_filename); + return result; + } +} + +static int trunk_binlog_fsync_ex(const bool bNeedLock, \ + const char *buff, int *length) +{ + int result; + int write_ret; + char full_filename[MAX_PATH_SIZE]; + + if (bNeedLock && (result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + if (*length == 0) //ignore + { + write_ret = 0; //skip + } + else if (write(trunk_binlog_fd, buff, *length) != *length) + { + logError("file: "__FILE__", line: %d, " \ + "write to binlog file \"%s\" fail, fd=%d, " \ + "errno: %d, error info: %s", \ + __LINE__, get_trunk_binlog_filename(full_filename), \ + trunk_binlog_fd, errno, STRERROR(errno)); + write_ret = errno != 0 ? errno : EIO; + } + else if (fsync(trunk_binlog_fd) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "sync to binlog file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, get_trunk_binlog_filename(full_filename), \ + errno, STRERROR(errno)); + write_ret = errno != 0 ? errno : EIO; + } + else + { + write_ret = 0; + } + + if (write_ret == 0) + { + trunk_binlog_write_version++; + *length = 0; //reset cache buff + } + + if (bNeedLock && (result=pthread_mutex_unlock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return write_ret; +} + +int trunk_binlog_write(const int timestamp, const char op_type, \ + const FDFSTrunkFullInfo *pTrunk) +{ + int result; + int write_ret; + + if ((result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + trunk_binlog_write_cache_len += sprintf(trunk_binlog_write_cache_buff + \ + trunk_binlog_write_cache_len, \ + "%d %c %d %d %d %d %d %d\n", \ + timestamp, op_type, \ + pTrunk->path.store_path_index, \ + pTrunk->path.sub_path_high, \ + pTrunk->path.sub_path_low, \ + pTrunk->file.id, \ + pTrunk->file.offset, \ + pTrunk->file.size); + + //check if buff full + if (TRUNK_BINLOG_BUFFER_SIZE - trunk_binlog_write_cache_len < 128) + { + write_ret = trunk_binlog_fsync(false); //sync to disk + } + else + { + write_ret = 0; + } + + if ((result=pthread_mutex_unlock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return write_ret; +} + +int trunk_binlog_write_buffer(const char *buff, const int length) +{ + int result; + int write_ret; + + if ((result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + //check if buff full + if (TRUNK_BINLOG_BUFFER_SIZE - (trunk_binlog_write_cache_len + \ + length) < 128) + { + write_ret = trunk_binlog_fsync(false); //sync to disk + } + else + { + write_ret = 0; + } + + if (write_ret == 0) + { + if (length >= TRUNK_BINLOG_BUFFER_SIZE) + { + if (trunk_binlog_write_cache_len > 0) + { + write_ret = trunk_binlog_fsync(false); + } + + if (write_ret == 0) + { + int len; + len = length; + write_ret = trunk_binlog_fsync_ex(false, \ + buff, &len); + } + } + else + { + memcpy(trunk_binlog_write_cache_buff + \ + trunk_binlog_write_cache_len, buff, length); + trunk_binlog_write_cache_len += length; + } + } + if ((result=pthread_mutex_unlock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return write_ret; +} + +static char *get_binlog_readable_filename(const void *pArg, \ + char *full_filename) +{ + static char buff[MAX_PATH_SIZE]; + + if (full_filename == NULL) + { + full_filename = buff; + } + + snprintf(full_filename, MAX_PATH_SIZE, + "%s/data/"TRUNK_DIR_NAME"/"TRUNK_SYNC_BINLOG_FILENAME, \ + g_fdfs_base_path); + return full_filename; +} + +int trunk_open_readable_binlog(TrunkBinLogReader *pReader, \ + get_filename_func filename_func, const void *pArg) +{ + char full_filename[MAX_PATH_SIZE]; + + if (pReader->binlog_fd >= 0) + { + close(pReader->binlog_fd); + } + + filename_func(pArg, full_filename); + pReader->binlog_fd = open(full_filename, O_RDONLY); + if (pReader->binlog_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open binlog file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (pReader->binlog_offset > 0 && \ + lseek(pReader->binlog_fd, pReader->binlog_offset, SEEK_SET) < 0) + { + logError("file: "__FILE__", line: %d, " \ + "seek binlog file \"%s\" fail, file offset=" \ + INT64_PRINTF_FORMAT", errno: %d, error info: %s", \ + __LINE__, full_filename, pReader->binlog_offset, \ + errno, STRERROR(errno)); + + close(pReader->binlog_fd); + pReader->binlog_fd = -1; + return errno != 0 ? errno : ESPIPE; + } + + return 0; +} + +static char *trunk_get_mark_filename_by_id_and_port(const char *storage_id, \ + const int port, char *full_filename, const int filename_size) +{ + if (g_use_storage_id) + { + snprintf(full_filename, filename_size, \ + "%s/data/"TRUNK_DIR_NAME"/%s%s", g_fdfs_base_path, \ + storage_id, TRUNK_SYNC_MARK_FILE_EXT); + } + else + { + snprintf(full_filename, filename_size, \ + "%s/data/"TRUNK_DIR_NAME"/%s_%d%s", g_fdfs_base_path, \ + storage_id, port, TRUNK_SYNC_MARK_FILE_EXT); + } + + return full_filename; +} + +static char *trunk_get_mark_filename_by_ip_and_port(const char *ip_addr, \ + const int port, char *full_filename, const int filename_size) +{ + snprintf(full_filename, filename_size, \ + "%s/data/"TRUNK_DIR_NAME"/%s_%d%s", g_fdfs_base_path, \ + ip_addr, port, TRUNK_SYNC_MARK_FILE_EXT); + + return full_filename; +} + +char *trunk_mark_filename_by_reader(const void *pArg, char *full_filename) +{ + const TrunkBinLogReader *pReader; + static char buff[MAX_PATH_SIZE]; + + pReader = (const TrunkBinLogReader *)pArg; + if (full_filename == NULL) + { + full_filename = buff; + } + + return trunk_get_mark_filename_by_id_and_port(pReader->storage_id, \ + g_server_port, full_filename, MAX_PATH_SIZE); +} + +static char *trunk_get_mark_filename_by_id(const char *storage_id, + char *full_filename, const int filename_size) +{ + return trunk_get_mark_filename_by_id_and_port(storage_id, g_server_port, \ + full_filename, filename_size); +} + +int trunk_reader_init(FDFSStorageBrief *pStorage, TrunkBinLogReader *pReader) +{ + char full_filename[MAX_PATH_SIZE]; + IniContext iniContext; + int result; + int64_t saved_binlog_offset; + bool bFileExist; + + saved_binlog_offset = pReader->binlog_offset; + + memset(pReader, 0, sizeof(TrunkBinLogReader)); + pReader->mark_fd = -1; + pReader->binlog_fd = -1; + + pReader->binlog_buff.buffer = (char *)malloc( \ + TRUNK_BINLOG_BUFFER_SIZE); + if (pReader->binlog_buff.buffer == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, TRUNK_BINLOG_BUFFER_SIZE, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + pReader->binlog_buff.current = pReader->binlog_buff.buffer; + + if (pStorage == NULL) + { + strcpy(pReader->storage_id, "0.0.0.0"); + } + else + { + strcpy(pReader->storage_id, pStorage->id); + } + trunk_mark_filename_by_reader(pReader, full_filename); + + if (pStorage == NULL) + { + bFileExist = false; + pReader->binlog_offset = saved_binlog_offset; + } + else + { + bFileExist = fileExists(full_filename); + if (!bFileExist && (g_use_storage_id && pStorage != NULL)) + { + char old_mark_filename[MAX_PATH_SIZE]; + trunk_get_mark_filename_by_ip_and_port( \ + pStorage->ip_addr, g_server_port, \ + old_mark_filename, sizeof(old_mark_filename)); + if (fileExists(old_mark_filename)) + { + if (rename(old_mark_filename, full_filename)!=0) + { + logError("file: "__FILE__", line: %d, "\ + "rename file %s to %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, old_mark_filename, \ + full_filename, errno, \ + STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + bFileExist = true; + } + } + } + + if (bFileExist) + { + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(full_filename, &iniContext)) \ + != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load from mark file \"%s\" fail, " \ + "error code: %d", \ + __LINE__, full_filename, result); + return result; + } + + if (iniContext.global.count < 1) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", item count: %d < 7", \ + __LINE__, full_filename, iniContext.global.count); + return ENOENT; + } + + pReader->binlog_offset = iniGetInt64Value(NULL, \ + MARK_ITEM_BINLOG_FILE_OFFSET, \ + &iniContext, -1); + if (pReader->binlog_offset < 0) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in mark file \"%s\", binlog_offset: "\ + INT64_PRINTF_FORMAT" < 0", \ + __LINE__, full_filename, \ + pReader->binlog_offset); + return EINVAL; + } + + iniFreeContext(&iniContext); + } + + pReader->last_binlog_offset = pReader->binlog_offset; + + pReader->mark_fd = open(full_filename, O_WRONLY | O_CREAT, 0644); + if (pReader->mark_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open mark file \"%s\" fail, " \ + "error no: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (!bFileExist && pStorage != NULL) + { + if ((result=trunk_write_to_mark_file(pReader)) != 0) + { + return result; + } + } + + if ((result=trunk_open_readable_binlog(pReader, \ + get_binlog_readable_filename, pReader)) != 0) + { + return result; + } + + result = trunk_binlog_preread(pReader); + if (result != 0 && result != ENOENT) + { + return result; + } + + return 0; +} + +void trunk_reader_destroy(TrunkBinLogReader *pReader) +{ + if (pReader->mark_fd >= 0) + { + close(pReader->mark_fd); + pReader->mark_fd = -1; + } + + if (pReader->binlog_fd >= 0) + { + close(pReader->binlog_fd); + pReader->binlog_fd = -1; + } + + if (pReader->binlog_buff.buffer != NULL) + { + free(pReader->binlog_buff.buffer); + pReader->binlog_buff.buffer = NULL; + pReader->binlog_buff.current = NULL; + pReader->binlog_buff.length = 0; + } +} + +static int trunk_write_to_mark_file(TrunkBinLogReader *pReader) +{ + char buff[128]; + int len; + int result; + + len = sprintf(buff, \ + "%s="INT64_PRINTF_FORMAT"\n", \ + MARK_ITEM_BINLOG_FILE_OFFSET, pReader->binlog_offset); + + if ((result=storage_write_to_fd(pReader->mark_fd, \ + trunk_mark_filename_by_reader, pReader, buff, len)) == 0) + { + pReader->last_binlog_offset = pReader->binlog_offset; + } + + return result; +} + +static int trunk_binlog_preread(TrunkBinLogReader *pReader) +{ + int bytes_read; + int saved_trunk_binlog_write_version; + + if (pReader->binlog_buff.version == trunk_binlog_write_version && \ + pReader->binlog_buff.length == 0) + { + return ENOENT; + } + + if (pReader->binlog_buff.length == TRUNK_BINLOG_BUFFER_SIZE) //buff full + { + return 0; + } + + saved_trunk_binlog_write_version = trunk_binlog_write_version; + if (pReader->binlog_buff.current != pReader->binlog_buff.buffer) + { + if (pReader->binlog_buff.length > 0) + { + memcpy(pReader->binlog_buff.buffer, \ + pReader->binlog_buff.current, \ + pReader->binlog_buff.length); + } + + pReader->binlog_buff.current = pReader->binlog_buff.buffer; + } + + bytes_read = read(pReader->binlog_fd, pReader->binlog_buff.buffer \ + + pReader->binlog_buff.length, \ + TRUNK_BINLOG_BUFFER_SIZE - pReader->binlog_buff.length); + if (bytes_read < 0) + { + logError("file: "__FILE__", line: %d, " \ + "read from binlog file \"%s\" fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "error no: %d, error info: %s", __LINE__, \ + get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset + pReader->binlog_buff.length, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + else if (bytes_read == 0) //end of binlog file + { + pReader->binlog_buff.version = saved_trunk_binlog_write_version; + return (pReader->binlog_buff.length == 0) ? ENOENT : 0; + } + + pReader->binlog_buff.length += bytes_read; + return 0; +} + +static int trunk_binlog_do_line_read(TrunkBinLogReader *pReader, \ + char *line, const int line_size, int *line_length) +{ + char *pLineEnd; + + if (pReader->binlog_buff.length == 0) + { + return ENOENT; + } + + pLineEnd = (char *)memchr(pReader->binlog_buff.current, '\n', \ + pReader->binlog_buff.length); + if (pLineEnd == NULL) + { + return ENOENT; + } + + *line_length = (pLineEnd - pReader->binlog_buff.current) + 1; + if (*line_length >= line_size) + { + logError("file: "__FILE__", line: %d, " \ + "read from binlog file \"%s\" fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "line buffer size: %d is too small! " \ + "<= line length: %d", __LINE__, \ + get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset, line_size, *line_length); + return ENOSPC; + } + + memcpy(line, pReader->binlog_buff.current, *line_length); + *(line + *line_length) = '\0'; + + pReader->binlog_buff.current = pLineEnd + 1; + pReader->binlog_buff.length -= *line_length; + + return 0; +} + +static int trunk_binlog_read_line(TrunkBinLogReader *pReader, \ + char *line, const int line_size, int *line_length) +{ + int result; + + result = trunk_binlog_do_line_read(pReader, line, \ + line_size, line_length); + if (result != ENOENT) + { + return result; + } + + result = trunk_binlog_preread(pReader); + if (result != 0) + { + return result; + } + + return trunk_binlog_do_line_read(pReader, line, \ + line_size, line_length); +} + +int trunk_binlog_read(TrunkBinLogReader *pReader, \ + TrunkBinLogRecord *pRecord, int *record_length) +{ +#define COL_COUNT 8 + char line[TRUNK_BINLOG_LINE_SIZE]; + char *cols[COL_COUNT]; + int result; + + result = trunk_binlog_read_line(pReader, line, \ + sizeof(line), record_length); + if (result != 0) + { + return result; + } + + if ((result=splitEx(line, ' ', cols, COL_COUNT)) < COL_COUNT) + { + logError("file: "__FILE__", line: %d, " \ + "read data from binlog file \"%s\" fail, " \ + "file offset: "INT64_PRINTF_FORMAT", " \ + "read item count: %d < %d", \ + __LINE__, get_binlog_readable_filename(pReader, NULL), \ + pReader->binlog_offset, result, COL_COUNT); + return ENOENT; + } + + pRecord->timestamp = atoi(cols[0]); + pRecord->op_type = *(cols[1]); + pRecord->trunk.path.store_path_index = atoi(cols[2]); + pRecord->trunk.path.sub_path_high = atoi(cols[3]); + pRecord->trunk.path.sub_path_low = atoi(cols[4]); + pRecord->trunk.file.id = atoi(cols[5]); + pRecord->trunk.file.offset = atoi(cols[6]); + pRecord->trunk.file.size = atoi(cols[7]); + + return 0; +} + +int trunk_unlink_mark_file(const char *storage_id) +{ + char old_filename[MAX_PATH_SIZE]; + char new_filename[MAX_PATH_SIZE]; + time_t t; + struct tm tm; + + t = g_current_time; + localtime_r(&t, &tm); + + trunk_get_mark_filename_by_id(storage_id, old_filename, \ + sizeof(old_filename)); + if (!fileExists(old_filename)) + { + return ENOENT; + } + + snprintf(new_filename, sizeof(new_filename), \ + "%s.%04d%02d%02d%02d%02d%02d", old_filename, \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec); + if (rename(old_filename, new_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file %s to %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, old_filename, new_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +int trunk_rename_mark_file(const char *old_ip_addr, const int old_port, \ + const char *new_ip_addr, const int new_port) +{ + char old_filename[MAX_PATH_SIZE]; + char new_filename[MAX_PATH_SIZE]; + + trunk_get_mark_filename_by_id_and_port(old_ip_addr, old_port, \ + old_filename, sizeof(old_filename)); + if (!fileExists(old_filename)) + { + return ENOENT; + } + + trunk_get_mark_filename_by_id_and_port(new_ip_addr, new_port, \ + new_filename, sizeof(new_filename)); + if (fileExists(new_filename)) + { + logWarning("file: "__FILE__", line: %d, " \ + "mark file %s already exists, " \ + "ignore rename file %s to %s", \ + __LINE__, new_filename, old_filename, new_filename); + return EEXIST; + } + + if (rename(old_filename, new_filename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file %s to %s fail" \ + ", errno: %d, error info: %s", \ + __LINE__, old_filename, new_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + return 0; +} + +static void trunk_sync_thread_exit(ConnectionInfo *pStorage) +{ + int result; + int i; + pthread_t tid; + + if ((result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + tid = pthread_self(); + for (i=0; iip_addr, pStorage->port); +} + +static int trunk_sync_data(TrunkBinLogReader *pReader, \ + ConnectionInfo *pStorage) +{ + int length; + char *p; + int result; + TrackerHeader header; + char in_buff[1]; + char *pBuff; + int64_t in_bytes; + + p = pReader->binlog_buff.buffer + pReader->binlog_buff.length - 1; + while (p != pReader->binlog_buff.buffer && *p != '\n') + { + p--; + } + + length = p - pReader->binlog_buff.buffer; + if (length == 0) + { + logWarning("FILE: "__FILE__", line: %d, " \ + "no buffer to sync, buffer length: %d, " \ + "should try again later", __LINE__, \ + pReader->binlog_buff.length); + return ENOENT; + } + length++; + + memset(&header, 0, sizeof(header)); + long2buff(length, header.pkg_len); + header.cmd = STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG; + if ((result=tcpsenddata_nb(pStorage->sock, &header, \ + sizeof(TrackerHeader), g_fdfs_network_timeout)) != 0) + { + logError("FILE: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorage->ip_addr, pStorage->port, \ + result, STRERROR(result)); + return result; + } + + if ((result=tcpsenddata_nb(pStorage->sock, pReader->binlog_buff.buffer,\ + length, g_fdfs_network_timeout)) != 0) + { + logError("FILE: "__FILE__", line: %d, " \ + "send data to storage server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pStorage->ip_addr, pStorage->port, \ + result, STRERROR(result)); + return result; + } + + pBuff = in_buff; + if ((result=fdfs_recv_response(pStorage, &pBuff, 0, &in_bytes)) != 0) + { + return result; + } + + pReader->binlog_offset += length; + pReader->binlog_buff.length -= length; + if (pReader->binlog_buff.length > 0) + { + pReader->binlog_buff.current = pReader->binlog_buff.buffer + length; + } + + return 0; +} + +static void* trunk_sync_thread_entrance(void* arg) +{ + FDFSStorageBrief *pStorage; + TrunkBinLogReader reader; + ConnectionInfo storage_server; + char local_ip_addr[IP_ADDRESS_SIZE]; + int read_result; + int sync_result; + int conn_result; + int result; + int previousCode; + int nContinuousFail; + time_t current_time; + time_t last_keep_alive_time; + + memset(local_ip_addr, 0, sizeof(local_ip_addr)); + memset(&reader, 0, sizeof(reader)); + reader.mark_fd = -1; + reader.binlog_fd = -1; + + current_time = g_current_time; + last_keep_alive_time = 0; + + pStorage = (FDFSStorageBrief *)arg; + + strcpy(storage_server.ip_addr, pStorage->ip_addr); + storage_server.port = g_server_port; + storage_server.sock = -1; + + logInfo("file: "__FILE__", line: %d, " \ + "trunk sync thread to storage server %s:%d started", \ + __LINE__, storage_server.ip_addr, storage_server.port); + + while (g_continue_flag && g_if_trunker_self && \ + pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ + pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ + pStorage->status != FDFS_STORAGE_STATUS_NONE) + { + previousCode = 0; + nContinuousFail = 0; + conn_result = 0; + while (g_continue_flag && g_if_trunker_self && \ + pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ + pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ + pStorage->status != FDFS_STORAGE_STATUS_NONE) + { + strcpy(storage_server.ip_addr, pStorage->ip_addr); + storage_server.sock = \ + socket(AF_INET, SOCK_STREAM, 0); + if(storage_server.sock < 0) + { + logCrit("file: "__FILE__", line: %d," \ + " socket create fail, " \ + "errno: %d, error info: %s. " \ + "program exit!", __LINE__, \ + errno, STRERROR(errno)); + g_continue_flag = false; + break; + } + + if (g_client_bind_addr && *g_bind_addr != '\0') + { + socketBind(storage_server.sock, g_bind_addr, 0); + } + + if (tcpsetnonblockopt(storage_server.sock) != 0) + { + nContinuousFail++; + close(storage_server.sock); + storage_server.sock = -1; + sleep(1); + + continue; + } + + if ((conn_result=connectserverbyip_nb(storage_server.sock,\ + pStorage->ip_addr, g_server_port, \ + g_fdfs_connect_timeout)) == 0) + { + char szFailPrompt[64]; + if (nContinuousFail == 0) + { + *szFailPrompt = '\0'; + } + else + { + sprintf(szFailPrompt, \ + ", continuous fail count: %d", \ + nContinuousFail); + } + logInfo("file: "__FILE__", line: %d, " \ + "successfully connect to " \ + "storage server %s:%d%s", __LINE__, \ + pStorage->ip_addr, g_server_port, \ + szFailPrompt); + nContinuousFail = 0; + break; + } + + if (previousCode != conn_result) + { + logError("file: "__FILE__", line: %d, " \ + "connect to storage server %s:%d fail" \ + ", errno: %d, error info: %s", \ + __LINE__, \ + pStorage->ip_addr, g_server_port, \ + conn_result, STRERROR(conn_result)); + previousCode = conn_result; + } + + nContinuousFail++; + close(storage_server.sock); + storage_server.sock = -1; + + if (!g_continue_flag) + { + break; + } + + sleep(1); + } + + if (nContinuousFail > 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to storage server %s:%d fail, " \ + "try count: %d, errno: %d, error info: %s", \ + __LINE__, pStorage->ip_addr, \ + g_server_port, nContinuousFail, \ + conn_result, STRERROR(conn_result)); + } + + if ((!g_continue_flag) || (!g_if_trunker_self) || \ + pStorage->status == FDFS_STORAGE_STATUS_DELETED || \ + pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + pStorage->status == FDFS_STORAGE_STATUS_NONE) + { + logError("file: "__FILE__", line: %d, break loop." \ + "g_continue_flag: %d, g_if_trunker_self: %d, " \ + "dest storage status: %d", __LINE__, \ + g_continue_flag, g_if_trunker_self, \ + pStorage->status); + break; + } + + if ((result=trunk_reader_init(pStorage, &reader)) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "trunk_reader_init fail, errno=%d, " \ + "program exit!", \ + __LINE__, result); + g_continue_flag = false; + break; + } + + getSockIpaddr(storage_server.sock, \ + local_ip_addr, IP_ADDRESS_SIZE); + insert_into_local_host_ip(local_ip_addr); + + /* + //printf("file: "__FILE__", line: %d, " \ + "storage_server.ip_addr=%s, " \ + "local_ip_addr: %s\n", \ + __LINE__, pStorage->ip_addr, local_ip_addr); + */ + + if (is_local_host_ip(pStorage->ip_addr)) + { //can't self sync to self + logError("file: "__FILE__", line: %d, " \ + "ip_addr %s belong to the local host," \ + " trunk sync thread exit.", \ + __LINE__, pStorage->ip_addr); + fdfs_quit(&storage_server); + close(storage_server.sock); + break; + } + + if (reader.binlog_offset == 0) + { + if (fdfs_deal_no_body_cmd(&storage_server, \ + STORAGE_PROTO_CMD_TRUNK_TRUNCATE_BINLOG_FILE) != 0) + { + close(storage_server.sock); + trunk_reader_destroy(&reader); + sleep(5); + continue; + } + } + + sync_result = 0; + while (g_continue_flag && \ + pStorage->status != FDFS_STORAGE_STATUS_DELETED && \ + pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \ + pStorage->status != FDFS_STORAGE_STATUS_NONE) + { + read_result = trunk_binlog_preread(&reader); + if (read_result == ENOENT) + { + if (reader.last_binlog_offset != \ + reader.binlog_offset) + { + if (trunk_write_to_mark_file(&reader)!=0) + { + logCrit("file: "__FILE__", line: %d, " \ + "trunk_write_to_mark_file fail, " \ + "program exit!", __LINE__); + g_continue_flag = false; + break; + } + } + + current_time = g_current_time; + if (current_time - last_keep_alive_time >= \ + g_heart_beat_interval) + { + if (fdfs_active_test(&storage_server)!=0) + { + break; + } + + last_keep_alive_time = current_time; + } + + if (!g_if_trunker_self) + { + break; + } + + usleep(g_sync_wait_usec); + continue; + } + + if (read_result != 0) + { + sleep(5); + continue; + } + + if ((sync_result=trunk_sync_data(&reader, \ + &storage_server)) != 0) + { + break; + } + + if (g_sync_interval > 0) + { + usleep(g_sync_interval); + } + } + + if (reader.last_binlog_offset != reader.binlog_offset) + { + if (trunk_write_to_mark_file(&reader) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "trunk_write_to_mark_file fail, " \ + "program exit!", __LINE__); + g_continue_flag = false; + break; + } + } + + close(storage_server.sock); + storage_server.sock = -1; + trunk_reader_destroy(&reader); + + if (!g_continue_flag) + { + break; + } + + if (!(sync_result == ENOTCONN || sync_result == EIO)) + { + sleep(1); + } + } + + if (storage_server.sock >= 0) + { + close(storage_server.sock); + } + trunk_reader_destroy(&reader); + + trunk_sync_thread_exit(&storage_server); + + return NULL; +} + +int trunk_sync_thread_start_all() +{ + FDFSStorageServer *pServer; + FDFSStorageServer *pEnd; + int result; + int ret; + + result = 0; + pEnd = g_storage_servers + g_storage_count; + for (pServer=g_storage_servers; pServerserver)); + if (ret != 0) + { + result = ret; + } + } + + return result; +} + +int trunk_sync_thread_start(const FDFSStorageBrief *pStorage) +{ + int result; + pthread_attr_t pattr; + pthread_t tid; + + if (pStorage->status == FDFS_STORAGE_STATUS_DELETED || \ + pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + pStorage->status == FDFS_STORAGE_STATUS_NONE) + { + return 0; + } + + if (is_local_host_ip(pStorage->ip_addr)) //can't self sync to self + { + return 0; + } + + if ((result=init_pthread_attr(&pattr, g_thread_stack_size)) != 0) + { + return result; + } + + /* + //printf("start storage ip_addr: %s, g_trunk_sync_thread_count=%d\n", + pStorage->ip_addr, g_trunk_sync_thread_count); + */ + + if ((result=pthread_create(&tid, &pattr, trunk_sync_thread_entrance, \ + (void *)pStorage)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "create thread failed, errno: %d, " \ + "error info: %s", \ + __LINE__, result, STRERROR(result)); + + pthread_attr_destroy(&pattr); + return result; + } + + if ((result=pthread_mutex_lock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + g_trunk_sync_thread_count++; + trunk_sync_tids = (pthread_t *)realloc(trunk_sync_tids, sizeof(pthread_t) * \ + g_trunk_sync_thread_count); + if (trunk_sync_tids == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, (int)sizeof(pthread_t) * \ + g_trunk_sync_thread_count, \ + errno, STRERROR(errno)); + } + else + { + trunk_sync_tids[g_trunk_sync_thread_count - 1] = tid; + } + + if ((result=pthread_mutex_unlock(&trunk_sync_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + pthread_attr_destroy(&pattr); + + return 0; +} + +int trunk_unlink_all_mark_files() +{ + FDFSStorageServer *pStorageServer; + FDFSStorageServer *pServerEnd; + int result; + + pServerEnd = g_storage_servers + g_storage_count; + for (pStorageServer=g_storage_servers; pStorageServerserver))) + { + continue; + } + + if ((result=trunk_unlink_mark_file( \ + pStorageServer->server.id)) != 0) + { + if (result != ENOENT) + { + return result; + } + } + } + + return 0; +} + diff --git a/storage/trunk_mgr/trunk_sync.h b/storage/trunk_mgr/trunk_sync.h new file mode 100644 index 0000000..015392c --- /dev/null +++ b/storage/trunk_mgr/trunk_sync.h @@ -0,0 +1,87 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//trunk_sync.h + +#ifndef _TRUNK_SYNC_H_ +#define _TRUNK_SYNC_H_ + +#include "tracker_types.h" +#include "storage_func.h" +#include "trunk_mem.h" + +#define TRUNK_OP_TYPE_ADD_SPACE 'A' +#define TRUNK_OP_TYPE_DEL_SPACE 'D' + +#define TRUNK_BINLOG_BUFFER_SIZE (64 * 1024) +#define TRUNK_BINLOG_LINE_SIZE 128 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct +{ + char storage_id[FDFS_STORAGE_ID_MAX_SIZE]; + BinLogBuffer binlog_buff; + int mark_fd; + int binlog_fd; + int64_t binlog_offset; + int64_t last_binlog_offset; //for write to mark file +} TrunkBinLogReader; + +typedef struct +{ + time_t timestamp; + char op_type; + FDFSTrunkFullInfo trunk; +} TrunkBinLogRecord; + +extern int g_trunk_sync_thread_count; + +int trunk_sync_init(); +int trunk_sync_destroy(); + +int trunk_binlog_write_buffer(const char *buff, const int length); + +int trunk_binlog_write(const int timestamp, const char op_type, \ + const FDFSTrunkFullInfo *pTrunk); + +int trunk_binlog_truncate(); + +int trunk_binlog_read(TrunkBinLogReader *pReader, \ + TrunkBinLogRecord *pRecord, int *record_length); + +int trunk_sync_thread_start_all(); +int trunk_sync_thread_start(const FDFSStorageBrief *pStorage); +int kill_trunk_sync_threads(); +int trunk_binlog_sync_func(void *args); + +char *get_trunk_binlog_filename(char *full_filename); +char *trunk_mark_filename_by_reader(const void *pArg, char *full_filename); +int trunk_unlink_all_mark_files(); +int trunk_unlink_mark_file(const char *storage_id); +int trunk_rename_mark_file(const char *old_ip_addr, const int old_port, \ + const char *new_ip_addr, const int new_port); + +int trunk_open_readable_binlog(TrunkBinLogReader *pReader, \ + get_filename_func filename_func, const void *pArg); + +int trunk_reader_init(FDFSStorageBrief *pStorage, TrunkBinLogReader *pReader); +void trunk_reader_destroy(TrunkBinLogReader *pReader); + +//trunk binlog compress +int trunk_binlog_compress_apply(); +int trunk_binlog_compress_commit(); +int trunk_binlog_compress_rollback(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..8cfb09d --- /dev/null +++ b/test/Makefile @@ -0,0 +1,28 @@ +.SUFFIXES: .c .o .lo + +COMPILE = $(CC) -g -Wall -O -D_FILE_OFFSET_BITS=64 -DDEBUG +INC_PATH = -I/usr/local/include -I/usr/local/include/fastdfs -I/usr/local/include/fastcommon +LIB_PATH = -L/usr/local/lib -lfdfsclient -lfastcommon +TARGET_PATH = $(TARGET_PREFIX)/bin + +#SHARED_OBJS = common_func.o dfs_func.o +SHARED_OBJS = common_func.o dfs_func_pc.o + +ALL_OBJS = $(SHARED_OBJS) + +ALL_PRGS = gen_files test_upload test_download test_delete combine_result + +all: $(ALL_OBJS) $(ALL_PRGS) +.o: + $(COMPILE) -o $@ $< $(SHARED_OBJS) $(LIB_PATH) $(INC_PATH) +.c: + $(COMPILE) -o $@ $< $(SHARED_OBJS) $(LIB_PATH) $(INC_PATH) +.c.o: + $(COMPILE) -c -o $@ $< $(INC_PATH) +.c.lo: + $(COMPILE) -c -fPIC -o $@ $< $(INC_PATH) +install: + mkdir -p $(TARGET_PATH) + cp -f $(ALL_PRGS) $(TARGET_PATH) +clean: + rm -f $(ALL_OBJS) $(ALL_PRGS) diff --git a/test/combine_result.c b/test/combine_result.c new file mode 100644 index 0000000..6af428a --- /dev/null +++ b/test/combine_result.c @@ -0,0 +1,270 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common_define.h" +#include "test_types.h" +#include "common_func.h" + +static int proccess_count; + +static int combine_stat_overall(int *ptotal_count, int *psuccess_count, int *ptime_used); +static int combine_stat_by(const char *file_prefix, EntryStat *stats, const int max_entries, int *entry_count); +static void print_stat_by(EntryStat *stats, const int entry_count); + +int main(int argc, char **argv) +{ + EntryStat stats[FILE_TYPE_COUNT]; + int entry_count; + int time_used; + int total_count; + int success_count; + int i; + int bytes; + int64_t total_bytes; + + if (argc < 2) + { + printf("Usage: %s \n", argv[0]); + return EINVAL; + } + + proccess_count = atoi(argv[1]); + if (proccess_count <= 0) + { + printf("Invalid proccess count: %d\n", proccess_count); + return EINVAL; + } + + total_count = 0; + success_count = 0; + time_used = 0; + combine_stat_overall(&total_count, &success_count, &time_used); + printf("total_count=%d, success_count=%d, success ratio: %.2f%% time_used=%ds, avg time used: %dms, QPS=%.2f\n\n", + total_count, success_count, total_count > 0 ? 100.00 * success_count / total_count : 0.00, + time_used, total_count > 0 ? time_used * 1000 / total_count : 0, + time_used == 0 ? 0 : (double)success_count / time_used); + + if (combine_stat_by(STAT_FILENAME_BY_FILE_TYPE, stats, FILE_TYPE_COUNT, &entry_count) == 0) + { + printf("file_type total_count success_count time_used(s) avg(ms) QPS success_ratio\n"); + print_stat_by(stats, entry_count); + printf("\n"); + } + + total_bytes = 0; + for (i=0; i 0) + { + printf("IO speed = %d KB\n", (int)(total_bytes / (time_used * 1024))); + } + + if (combine_stat_by(STAT_FILENAME_BY_STORAGE_IP, stats, FILE_TYPE_COUNT, &entry_count) == 0) + { + printf("ip_addr total_count success_count time_used(s) avg(ms) QPS success_ratio\n"); + print_stat_by(stats, entry_count); + printf("\n"); + } + + return 0; +} + +static void print_stat_by(EntryStat *stats, const int entry_count) +{ + EntryStat *pEntry; + EntryStat *pEnd; + int seconds; + + pEnd = stats + entry_count; + for (pEntry=stats; pEntrytime_used / 1000; + printf("%s %d %d %d %d %.2f %.2f\n", pEntry->id, pEntry->total_count, + pEntry->success_count, (int)(pEntry->time_used / 1000), + pEntry->total_count == 0 ? 0 : (int)(pEntry->time_used / pEntry->total_count), + seconds == 0 ? 0 : (double)pEntry->success_count / seconds, + pEntry->total_count > 0 ? 100.00 * pEntry->success_count / pEntry->total_count : 0.00); + } +} + +static int combine_stat_by(const char *file_prefix, EntryStat *stats, const int max_entries, int *entry_count) +{ + char filename[64]; + FILE *fp; + int proccess_index; + char buff[256]; + char id[64]; + int64_t time_used; + int total_count; + int success_count; + EntryStat *pEntry; + EntryStat *pEnd; + + *entry_count = 0; + memset(stats, 0, sizeof(EntryStat) * max_entries); + for (proccess_index=0; proccess_indexid) == 0) + { + break; + } + } + + if (pEntry == pEnd) //not found + { + if (*entry_count >= max_entries) + { + printf("entry count: %d >= max entries: %d\n", *entry_count, max_entries); + return ENOSPC; + } + + strcpy(pEntry->id, id); + (*entry_count)++; + } + + pEntry->total_count += total_count; + pEntry->success_count += success_count; + pEntry->time_used += time_used; + } + + fclose(fp); + } + + pEnd = stats + (*entry_count); + for (pEntry=stats; pEntrytime_used /= proccess_count; + } + + return 0; +} + +static int combine_stat_overall(int *ptotal_count, int *psuccess_count, int *ptime_used) +{ + char filename[64]; + FILE *fp; + int proccess_index; + char buff[256]; + int time_used; + int total_count; + int success_count; + + *ptotal_count = 0; + *psuccess_count = 0; + *ptime_used = 0; + + for (proccess_index=0; proccess_index +#include +#include +#include +#include +#include +#include +#include +#include "common_func.h" + +int getFileContent(const char *filename, char **buff, int64_t *file_size) +{ + int fd; + + fd = open(filename, O_RDONLY); + if (fd < 0) + { + *buff = NULL; + *file_size = 0; + return errno != 0 ? errno : ENOENT; + } + + if ((*file_size=lseek(fd, 0, SEEK_END)) < 0) + { + *buff = NULL; + *file_size = 0; + close(fd); + return errno != 0 ? errno : EIO; + } + + *buff = (char *)malloc(*file_size + 1); + if (*buff == NULL) + { + *file_size = 0; + close(fd); + + return errno != 0 ? errno : ENOMEM; + } + + if (lseek(fd, 0, SEEK_SET) < 0) + { + *buff = NULL; + *file_size = 0; + close(fd); + return errno != 0 ? errno : EIO; + } + if (read(fd, *buff, *file_size) != *file_size) + { + free(*buff); + *buff = NULL; + *file_size = 0; + close(fd); + return errno != 0 ? errno : EIO; + } + + (*buff)[*file_size] = '\0'; + close(fd); + + return 0; +} + diff --git a/test/common_func.h b/test/common_func.h new file mode 100644 index 0000000..bf6386a --- /dev/null +++ b/test/common_func.h @@ -0,0 +1,16 @@ +//common_func.h + +#ifndef _COMMON_FUNC_H +#define _COMMON_FUNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +int getFileContent(const char *filename, char **buff, int64_t *file_size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/dfs_func.c b/test/dfs_func.c new file mode 100644 index 0000000..f69e5ed --- /dev/null +++ b/test/dfs_func.c @@ -0,0 +1,143 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_global.h" +#include "dfs_func.h" +#include "fdfs_client.h" + +int dfs_init(const int proccess_index, const char *conf_filename) +{ + return fdfs_client_init(conf_filename); +} + +void dfs_destroy() +{ + fdfs_client_destroy(); +} + +static int downloadFileCallback(void *arg, const int64_t file_size, const char *data, \ + const int current_size) +{ + return 0; +} + +int upload_file(const char *file_buff, const int file_size, char *file_id, char *storage_ip) +{ + int result; + int store_path_index; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + ConnectionInfo storageServer; + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + return errno != 0 ? errno : ECONNREFUSED; + } + + *group_name = '\0'; + if ((result=tracker_query_storage_store(pTrackerServer, &storageServer, + group_name, &store_path_index)) != 0) + { + tracker_disconnect_server_ex(pTrackerServer, true); + return result; + } + + if ((pStorageServer=tracker_connect_server(&storageServer, &result)) \ + == NULL) + { + tracker_disconnect_server(pTrackerServer); + return result; + } + + strcpy(storage_ip, storageServer.ip_addr); + result = storage_upload_by_filebuff1(pTrackerServer, pStorageServer, + store_path_index, file_buff, file_size, NULL, NULL, 0, "", file_id); + + tracker_disconnect_server(pTrackerServer); + tracker_disconnect_server(pStorageServer); + + return result; +} + +int download_file(const char *file_id, int *file_size, char *storage_ip) +{ + int result; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + ConnectionInfo storageServer; + int64_t file_bytes; + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + return errno != 0 ? errno : ECONNREFUSED; + } + + if ((result=tracker_query_storage_fetch1(pTrackerServer, \ + &storageServer, file_id)) != 0) + { + tracker_disconnect_server_ex(pTrackerServer, true); + return result; + } + + if ((pStorageServer=tracker_connect_server(&storageServer, &result)) \ + == NULL) + { + tracker_disconnect_server(pTrackerServer); + return result; + } + + strcpy(storage_ip, storageServer.ip_addr); + result = storage_download_file_ex1(pTrackerServer, pStorageServer, \ + file_id, 0, 0, downloadFileCallback, NULL, &file_bytes); + *file_size = file_bytes; + + tracker_disconnect_server(pTrackerServer); + tracker_disconnect_server(pStorageServer); + + return result; +} + +int delete_file(const char *file_id, char *storage_ip) +{ + int result; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pStorageServer; + ConnectionInfo storageServer; + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + return errno != 0 ? errno : ECONNREFUSED; + } + + if ((result=tracker_query_storage_update1(pTrackerServer, \ + &storageServer, file_id)) != 0) + { + tracker_disconnect_server_ex(pTrackerServer, true); + return result; + } + + if ((pStorageServer=tracker_connect_server(&storageServer, &result)) \ + == NULL) + { + tracker_disconnect_server(pTrackerServer); + return result; + } + + strcpy(storage_ip, storageServer.ip_addr); + result = storage_delete_file1(pTrackerServer, pStorageServer, file_id); + + tracker_disconnect_server(pTrackerServer); + tracker_disconnect_server(pStorageServer); + + return result; +} + diff --git a/test/dfs_func.h b/test/dfs_func.h new file mode 100644 index 0000000..328b5a7 --- /dev/null +++ b/test/dfs_func.h @@ -0,0 +1,55 @@ +//dfs_func.h + +#ifndef _DFS_FUNC_H +#define _DFS_FUNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* +*init function +* param proccess_index the process index based 0 +* param conf_filename the config filename +* return 0 if success, none zero for error +*/ +int dfs_init(const int proccess_index, const char *conf_filename); + +/* +*destroy function +* return void +*/ +void dfs_destroy(); + +/* +* upload file to the storage server +* param file_buff the file content +* param file_size the file size (bytes) +* param file_id return the file id (max length 63) +* param storage_ip return the storage server ip address (max length 15) +* return 0 if success, none zero for error +*/ +int upload_file(const char *file_buff, const int file_size, char *file_id, char *storage_ip); + +/* +* download file from the storage server +* param file_id the file id +* param file_size return the file size (bytes) +* param storage_ip return the storage server ip address (max length 15) +* return 0 if success, none zero for error +*/ +int download_file(const char *file_id, int *file_size, char *storage_ip); + +/* +* delete file from the storage server +* param file_id the file id +* param storage_ip return the storage server ip address (max length 15) +* return 0 if success, none zero for error +*/ +int delete_file(const char *file_id, char *storage_ip); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/dfs_func_pc.c b/test/dfs_func_pc.c new file mode 100644 index 0000000..d1f6048 --- /dev/null +++ b/test/dfs_func_pc.c @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_global.h" +#include "dfs_func.h" +#include "fdfs_client.h" + +static ConnectionInfo *pTrackerServer; +static ConnectionInfo storage_servers[FDFS_MAX_SERVERS_EACH_GROUP]; +static int storage_server_count = 0; + +static ConnectionInfo *getConnectedStorageServer( + ConnectionInfo *pStorageServer, int *err_no) +{ + ConnectionInfo *pEnd; + ConnectionInfo *pServer; + + pEnd = storage_servers + storage_server_count; + for (pServer=storage_servers; pServerip_addr, pServer->ip_addr) == 0) + { + if (pServer->sock < 0) + { + *err_no = conn_pool_connect_server(pServer, \ + g_fdfs_connect_timeout); + if (*err_no != 0) + { + return NULL; + } + } + else + { + *err_no = 0; + } + + return pServer; + } + } + + pServer = pEnd; + memcpy(pServer, pStorageServer, sizeof(ConnectionInfo)); + pServer->sock = -1; + if ((*err_no=conn_pool_connect_server(pServer, \ + g_fdfs_connect_timeout)) != 0) + { + return NULL; + } + + storage_server_count++; + + *err_no = 0; + return pServer; +} + +int dfs_init(const int proccess_index, const char *conf_filename) +{ + int result; + if ((result=fdfs_client_init(conf_filename)) != 0) + { + return result; + } + + pTrackerServer = tracker_get_connection(); + if (pTrackerServer == NULL) + { + return errno != 0 ? errno : ECONNREFUSED; + } + + return 0; +} + +void dfs_destroy() +{ + ConnectionInfo *pEnd; + ConnectionInfo *pServer; + + tracker_disconnect_server(pTrackerServer); + + pEnd = storage_servers + storage_server_count; + for (pServer=storage_servers; pServer +#include +#include +#include +#include +#include "common_define.h" +#include "test_types.h" + +typedef struct { + int bytes; + char *filename; +} TestFileInfo; + +TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K"}, + {50 * 1024, "50K"}, + {200 * 1024, "200K"}, + {1 * 1024 * 1024, "1M"}, + {10 * 1024 * 1024, "10M"}, + {100 * 1024 * 1024, "100M"} +}; + +int main() +{ +#define BUFF_SIZE (1 * 1024) + int i; + int k; + int loop; + FILE *fp; + unsigned char buff[BUFF_SIZE]; + unsigned char *p; + unsigned char *pEnd; + + srand(SRAND_SEED); + pEnd = buff + BUFF_SIZE; + for (i=0; i +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "test_types.h" +#include "common_func.h" +#include "dfs_func.h" + +#define PROCESS_COUNT 10 + +typedef struct { + int file_type; //index + char *file_id; +} FileEntry; + +typedef struct { + int bytes; //file size + char *filename; + int count; //total file count + int delete_count; + int success_count; //success upload count + int64_t time_used; //unit: ms +} TestFileInfo; + +#ifdef DEBUG //for debug +static TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K", 1000 / PROCESS_COUNT, 0, 0, 0}, + {50 * 1024, "50K", 2000 / PROCESS_COUNT, 0, 0, 0}, + {200 * 1024, "200K", 1000 / PROCESS_COUNT, 0, 0, 0}, + {1 * 1024 * 1024, "1M", 200 / PROCESS_COUNT, 0, 0, 0}, + {10 * 1024 * 1024, "10M", 20 / PROCESS_COUNT, 0, 0, 0}, + {100 * 1024 * 1024, "100M", 10 / PROCESS_COUNT, 0, 0, 0} +}; + +#else + +static TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K", 1000000 / PROCESS_COUNT, 0, 0, 0}, + {50 * 1024, "50K", 2000000 / PROCESS_COUNT, 0, 0, 0}, + {200 * 1024, "200K", 1000000 / PROCESS_COUNT, 0, 0, 0}, + {1 * 1024 * 1024, "1M", 200000 / PROCESS_COUNT, 0, 0, 0}, + {10 * 1024 * 1024, "10M", 20000 / PROCESS_COUNT, 0, 0, 0}, + {100 * 1024 * 1024, "100M", 1000 / PROCESS_COUNT, 0, 0, 0} +}; + +#endif + +static StorageStat storages[MAX_STORAGE_COUNT]; +static int storage_count = 0; +static time_t start_time; +static int total_count = 0; +static int success_count = 0; +static FILE *fpFail = NULL; + +static int proccess_index = 0; +static int file_count = 0; +static FileEntry *file_entries = NULL; + +static int load_file_ids(); +static int test_init(); +static int save_stats_by_overall(); +static int save_stats_by_file_type(); +static int save_stats_by_storage_ip(); +static int add_to_storage_stat(const char *storage_ip, const int result, const int time_used); + +int main(int argc, char **argv) +{ + int result; + int i; + int file_type; + char storage_ip[IP_ADDRESS_SIZE]; + char *conf_filename; + struct timeval tv_start; + struct timeval tv_end; + int time_used; + + if (argc < 2) + { + printf("Usage: %s [config_filename]\n", argv[0]); + return EINVAL; + } + + log_init(); + proccess_index = atoi(argv[1]); + if (proccess_index < 0 || proccess_index >= PROCESS_COUNT) + { + printf("Invalid proccess index: %d\n", proccess_index); + return EINVAL; + } + + if (argc >= 3) + { + conf_filename = argv[2]; + } + else + { + conf_filename = "/etc/fdfs/client.conf"; + } + + if ((result = load_file_ids()) != 0) + { + return result; + } + + if ((result=test_init()) != 0) + { + return result; + } + + if ((result=dfs_init(proccess_index, conf_filename)) != 0) + { + return result; + } + +#ifndef WIN32 + if (daemon(1, 1) != 0) + { + return errno != 0 ? errno : EFAULT; + } +#endif + + /* + printf("file_count = %d\n", file_count); + printf("file_entries[0]=%s\n", file_entries[0].file_id); + printf("file_entries[%d]=%s\n", file_count-1, file_entries[file_count-1].file_id); + */ + + memset(&storages, 0, sizeof(storages)); + memset(storage_ip, 0, sizeof(storage_ip)); + + start_time = time(NULL); + result = 0; + total_count = 0; + success_count = 0; + for (i=0; iip_addr) == 0) + { + break; + } + } + + if (pStorage == pEnd) //not found + { + if (storage_count >= MAX_STORAGE_COUNT) + { + printf("storage_count %d >= %d\n", storage_count, MAX_STORAGE_COUNT); + return ENOSPC; + } + + strcpy(pStorage->ip_addr, storage_ip); + storage_count++; + } + + pStorage->time_used += time_used; + pStorage->total_count++; + if (result == 0) + { + pStorage->success_count++; + } + + return 0; +} + +static int get_file_type_index(const int file_bytes) +{ + TestFileInfo *pFile; + TestFileInfo *pEnd; + + pEnd = files + FILE_TYPE_COUNT; + for (pFile=files; pFilebytes) + { + return pFile - files; + } + } + + return -1; +} + +static int load_file_ids() +{ + int i; + int result; + int64_t file_size; + int bytes; + char filename[64]; + char *file_buff; + char *p; + int nLineCount; + char *pStart; + char *pEnd; + char *pFind; + + sprintf(filename, "upload/%s.%d", FILENAME_FILE_ID, proccess_index); + if ((result=getFileContent(filename, &file_buff, &file_size)) != 0) + { + printf("file: "__FILE__", line: %d, " + "getFileContent %s fail, errno: %d, error info: %s\n", __LINE__, + filename, errno, STRERROR(errno)); + + return result; + } + + nLineCount = 0; + p = file_buff; + while (*p != '\0') + { + if (*p == '\n') + { + nLineCount++; + } + + p++; + } + + file_count = nLineCount; + if (file_count == 0) + { + printf("file: "__FILE__", line: %d, " + "file count == 0 in file %s\n", __LINE__, filename); + free(file_buff); + return EINVAL; + } + + file_entries = (FileEntry *)malloc(sizeof(FileEntry) * file_count); + if (file_entries == NULL) + { + printf("file: "__FILE__", line: %d, " + "malloc %d bytes fail\n", __LINE__, \ + (int)sizeof(FileEntry) * file_count); + free(file_buff); + return ENOMEM; + } + memset(file_entries, 0, sizeof(FileEntry) * file_count); + + i = 0; + p = file_buff; + pStart = file_buff; + while (i < file_count) + { + if (*p == '\n') + { + *p = '\0'; + pFind = strchr(pStart, ' '); + if (pFind == NULL) + { + printf("file: "__FILE__", line: %d, " + "can't find ' ' in file %s\n", __LINE__, filename); + result = EINVAL; + break; + } + + pFind++; + pEnd = strchr(pFind, ' '); + if (pEnd == NULL) + { + printf("file: "__FILE__", line: %d, " + "can't find ' ' in file %s\n", __LINE__, filename); + result = EINVAL; + break; + } + *pEnd = '\0'; + bytes = atoi(pFind); + + pFind = pEnd + 1; //skip space + pEnd = strchr(pFind, ' '); + if (pEnd == NULL) + { + printf("file: "__FILE__", line: %d, " + "can't find ' ' in file %s\n", __LINE__, filename); + result = EINVAL; + break; + } + *pEnd = '\0'; + + file_entries[i].file_type = get_file_type_index(bytes); + if (file_entries[i].file_type < 0) + { + printf("file: "__FILE__", line: %d, " + "invalid file bytes: %d in file %s\n", __LINE__, bytes, filename); + result = EINVAL; + break; + } + + file_entries[i].file_id = strdup(pFind); + if (file_entries[i].file_id == NULL) + { + printf("file: "__FILE__", line: %d, " + "malloc %d bytes fail\n", __LINE__, \ + (int)strlen(pFind) + 1); + result = ENOMEM; + break; + } + + i++; + pStart = ++p; + } + else + { + p++; + } + } + + free(file_buff); + + return result; +} + +static int test_init() +{ + char filename[64]; + + if (access("delete", 0) != 0 && mkdir("delete", 0755) != 0) + { + } + + if (chdir("delete") != 0) + { + printf("chdir fail, errno: %d, error info: %s\n", errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + sprintf(filename, "%s.%d", FILENAME_FAIL, proccess_index); + if ((fpFail=fopen(filename, "wb")) == NULL) + { + printf("open file %s fail, errno: %d, error info: %s\n", + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + return 0; +} + diff --git a/test/test_delete.sh b/test/test_delete.sh new file mode 100644 index 0000000..4dd2c4b --- /dev/null +++ b/test/test_delete.sh @@ -0,0 +1,6 @@ +i=0 +while [ $i -lt 10 ]; do + ./test_delete $i & + let i=i+1 +done + diff --git a/test/test_download.c b/test/test_download.c new file mode 100644 index 0000000..92bd019 --- /dev/null +++ b/test/test_download.c @@ -0,0 +1,513 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "common_define.h" +#include "test_types.h" +#include "common_func.h" +#include "dfs_func.h" + +#define PROCESS_COUNT 20 + +#ifdef DEBUG //for debug +#define TOTAL_SECONDS 300 +#else +#define TOTAL_SECONDS 8 * 3600 +#endif + +typedef struct { + int file_type; //index + char *file_id; +} FileEntry; + +typedef struct { + int bytes; //file size + char *filename; + int count; //total file count + int download_count; + int success_count; //success upload count + int64_t time_used; //unit: ms +} TestFileInfo; + +#ifdef DEBUG //for debug +static TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K", 1000 / PROCESS_COUNT, 0, 0, 0}, + {50 * 1024, "50K", 2000 / PROCESS_COUNT, 0, 0, 0}, + {200 * 1024, "200K", 1000 / PROCESS_COUNT, 0, 0, 0}, + {1 * 1024 * 1024, "1M", 200 / PROCESS_COUNT, 0, 0, 0}, + {10 * 1024 * 1024, "10M", 20 / PROCESS_COUNT, 0, 0, 0}, + {100 * 1024 * 1024, "100M", 10 / PROCESS_COUNT, 0, 0, 0} +}; + +#else + +static TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K", 1000000 / PROCESS_COUNT, 0, 0, 0}, + {50 * 1024, "50K", 2000000 / PROCESS_COUNT, 0, 0, 0}, + {200 * 1024, "200K", 1000000 / PROCESS_COUNT, 0, 0, 0}, + {1 * 1024 * 1024, "1M", 200000 / PROCESS_COUNT, 0, 0, 0}, + {10 * 1024 * 1024, "10M", 20000 / PROCESS_COUNT, 0, 0, 0}, + {100 * 1024 * 1024, "100M", 1000 / PROCESS_COUNT, 0, 0, 0} +}; + +#endif + +static StorageStat storages[MAX_STORAGE_COUNT]; +static int storage_count = 0; +static time_t start_time; +static int total_count = 0; +static int success_count = 0; +static FILE *fpFail = NULL; + +static int proccess_index = 0; +static int file_count = 0; +static FileEntry *file_entries = NULL; + +static int load_file_ids(); +static int test_init(); +static int save_stats_by_overall(); +static int save_stats_by_file_type(); +static int save_stats_by_storage_ip(); +static int add_to_storage_stat(const char *storage_ip, const int result, const int time_used); + +int main(int argc, char **argv) +{ + int result; + int file_index; + int file_type; + int file_size; + char *conf_filename; + char storage_ip[IP_ADDRESS_SIZE]; + struct timeval tv_start; + struct timeval tv_end; + int time_used; + + if (argc < 2) + { + printf("Usage: %s [config_filename]\n", argv[0]); + return EINVAL; + } + + log_init(); + proccess_index = atoi(argv[1]); + if (proccess_index < 0 || proccess_index >= PROCESS_COUNT) + { + printf("Invalid proccess index: %d\n", proccess_index); + return EINVAL; + } + + if (argc >= 3) + { + conf_filename = argv[2]; + } + else + { + conf_filename = "/etc/fdfs/client.conf"; + } + + if ((result = load_file_ids()) != 0) + { + return result; + } + + if ((result=test_init()) != 0) + { + return result; + } + + if ((result=dfs_init(proccess_index, conf_filename)) != 0) + { + return result; + } + +#ifndef WIN32 + if (daemon(1, 1) != 0) + { + return errno != 0 ? errno : EFAULT; + } +#endif + + /* + printf("file_count = %d\n", file_count); + printf("file_entries[0]=%s\n", file_entries[0].file_id); + printf("file_entries[%d]=%s\n", file_count-1, file_entries[file_count-1].file_id); + */ + + memset(&storages, 0, sizeof(storages)); + memset(storage_ip, 0, sizeof(storage_ip)); + + start_time = time(NULL); + srand(SRAND_SEED); + result = 0; + total_count = 0; + success_count = 0; + while (time(NULL) - start_time < TOTAL_SECONDS) + { + file_index = (int)(file_count * ((double)rand() / RAND_MAX)); + if (file_index >= file_count) + { + printf("file_index=%d!!!!\n", file_index); + continue; + } + + file_type = file_entries[file_index].file_type; + files[file_type].download_count++; + total_count++; + + gettimeofday(&tv_start, NULL); + *storage_ip = '\0'; + result = download_file(file_entries[file_index].file_id, &file_size, storage_ip); + gettimeofday(&tv_end, NULL); + time_used = TIME_SUB_MS(tv_end, tv_start); + files[file_type].time_used += time_used; + + add_to_storage_stat(storage_ip, result, time_used); + if (result == 0) //success + { + if (file_size != files[file_type].bytes) + { + result = EINVAL; + } + } + + if (result == 0) //success + { + success_count++; + files[file_type].success_count++; + } + else //fail + { + fprintf(fpFail, "%d %d %s %s %d %d\n", (int)tv_end.tv_sec, + files[file_type].bytes, file_entries[file_index].file_id, + storage_ip, result, time_used); + fflush(fpFail); + } + + if (total_count % 10000 == 0) + { + if ((result=save_stats_by_overall()) != 0) + { + break; + } + if ((result=save_stats_by_file_type()) != 0) + { + break; + } + + if ((result=save_stats_by_storage_ip()) != 0) + { + break; + } + } + } + + save_stats_by_overall(); + save_stats_by_file_type(); + save_stats_by_storage_ip(); + + fclose(fpFail); + + dfs_destroy(); + + printf("proccess %d, time used: %ds\n", proccess_index, (int)(time(NULL) - start_time)); + return result; +} + +static int save_stats_by_file_type() +{ + int k; + char filename[64]; + FILE *fp; + + sprintf(filename, "%s.%d", STAT_FILENAME_BY_FILE_TYPE, proccess_index); + if ((fp=fopen(filename, "wb")) == NULL) + { + printf("open file %s fail, errno: %d, error info: %s\n", + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + fprintf(fp, "#file_type total_count success_count time_used(ms)\n"); + for (k=0; kip_addr) == 0) + { + break; + } + } + + if (pStorage == pEnd) //not found + { + if (storage_count >= MAX_STORAGE_COUNT) + { + printf("storage_count %d >= %d\n", storage_count, MAX_STORAGE_COUNT); + return ENOSPC; + } + + strcpy(pStorage->ip_addr, storage_ip); + storage_count++; + } + + pStorage->time_used += time_used; + pStorage->total_count++; + if (result == 0) + { + pStorage->success_count++; + } + + return 0; +} + +static int get_file_type_index(const int file_bytes) +{ + TestFileInfo *pFile; + TestFileInfo *pEnd; + + pEnd = files + FILE_TYPE_COUNT; + for (pFile=files; pFilebytes) + { + return pFile - files; + } + } + + return -1; +} + +static int load_file_ids() +{ + int i; + int result; + int64_t file_size; + int bytes; + char filename[64]; + char *file_buff; + char *p; + int nLineCount; + int nSkipLines; + char *pStart; + char *pEnd; + char *pFind; + + sprintf(filename, "upload/%s.%d", FILENAME_FILE_ID, proccess_index / 2); + if ((result=getFileContent(filename, &file_buff, &file_size)) != 0) + { + printf("file: "__FILE__", line: %d, " + "getFileContent %s fail, errno: %d, error info: %s\n", __LINE__, + filename, errno, STRERROR(errno)); + + return result; + } + + nLineCount = 0; + p = file_buff; + while (*p != '\0') + { + if (*p == '\n') + { + nLineCount++; + } + + p++; + } + + file_count = nLineCount / 2; + if (file_count == 0) + { + printf("file: "__FILE__", line: %d, " + "file count == 0 in file %s\n", __LINE__, filename); + free(file_buff); + return EINVAL; + } + + file_entries = (FileEntry *)malloc(sizeof(FileEntry) * file_count); + if (file_entries == NULL) + { + printf("file: "__FILE__", line: %d, " + "malloc %d bytes fail\n", __LINE__, \ + (int)sizeof(FileEntry) * file_count); + free(file_buff); + return ENOMEM; + } + memset(file_entries, 0, sizeof(FileEntry) * file_count); + + nSkipLines = (proccess_index % 2) * file_count; + i = 0; + p = file_buff; + while (i < nSkipLines) + { + if (*p == '\n') + { + i++; + } + + p++; + } + + pStart = p; + i = 0; + while (i < file_count) + { + if (*p == '\n') + { + *p = '\0'; + pFind = strchr(pStart, ' '); + if (pFind == NULL) + { + printf("file: "__FILE__", line: %d, " + "can't find ' ' in file %s\n", __LINE__, filename); + result = EINVAL; + break; + } + + pFind++; + pEnd = strchr(pFind, ' '); + if (pEnd == NULL) + { + printf("file: "__FILE__", line: %d, " + "can't find ' ' in file %s\n", __LINE__, filename); + result = EINVAL; + break; + } + *pEnd = '\0'; + bytes = atoi(pFind); + + pFind = pEnd + 1; //skip space + pEnd = strchr(pFind, ' '); + if (pEnd == NULL) + { + printf("file: "__FILE__", line: %d, " + "can't find ' ' in file %s\n", __LINE__, filename); + result = EINVAL; + break; + } + *pEnd = '\0'; + + file_entries[i].file_type = get_file_type_index(bytes); + if (file_entries[i].file_type < 0) + { + printf("file: "__FILE__", line: %d, " + "invalid file bytes: %d in file %s\n", __LINE__, bytes, filename); + result = EINVAL; + break; + } + + file_entries[i].file_id = strdup(pFind); + if (file_entries[i].file_id == NULL) + { + printf("file: "__FILE__", line: %d, " + "malloc %d bytes fail\n", __LINE__, \ + (int)strlen(pFind) + 1); + result = ENOMEM; + break; + } + + i++; + pStart = ++p; + } + else + { + p++; + } + } + + free(file_buff); + + return result; +} + +static int test_init() +{ + char filename[64]; + + if (access("download", 0) != 0 && mkdir("download", 0755) != 0) + { + } + + if (chdir("download") != 0) + { + printf("chdir fail, errno: %d, error info: %s\n", errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + sprintf(filename, "%s.%d", FILENAME_FAIL, proccess_index); + if ((fpFail=fopen(filename, "wb")) == NULL) + { + printf("open file %s fail, errno: %d, error info: %s\n", + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + return 0; +} + diff --git a/test/test_download.sh b/test/test_download.sh new file mode 100644 index 0000000..f22e282 --- /dev/null +++ b/test/test_download.sh @@ -0,0 +1,6 @@ +i=0 +while [ $i -lt 20 ]; do + ./test_download $i & + let i=i+1 +done + diff --git a/test/test_types.h b/test/test_types.h new file mode 100644 index 0000000..4ff6fa2 --- /dev/null +++ b/test/test_types.h @@ -0,0 +1,43 @@ +//test_types.h + +#ifndef _TEST_TYPES_H +#define _TEST_TYPES_H + +#define FILE_TYPE_COUNT 6 +#define MAX_STORAGE_COUNT 5 + +#define STAT_FILENAME_BY_FILE_TYPE "stat_by_file_type" +#define STAT_FILENAME_BY_STORAGE_IP "stat_by_storage_ip" +#define STAT_FILENAME_BY_OVERALL "stat_by_overall" + +#define FILENAME_FILE_ID "file_id" +#define FILENAME_FAIL "fail" + +#define IP_ADDRESS_SIZE 16 +#define SRAND_SEED 1225420780 + +#define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000) + +typedef struct { + char ip_addr[IP_ADDRESS_SIZE]; + int total_count; + int success_count; + int64_t time_used; +} StorageStat; + +typedef struct { + char id[64]; + int total_count; + int success_count; + int64_t time_used; +} EntryStat; + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/test_upload.c b/test/test_upload.c new file mode 100644 index 0000000..9901b14 --- /dev/null +++ b/test/test_upload.c @@ -0,0 +1,420 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common_define.h" +#include "logger.h" +#include "test_types.h" +#include "common_func.h" +#include "dfs_func.h" + +#define PROCESS_COUNT 10 + +typedef struct { + int bytes; //file size + char *filename; + int count; //total file count + int upload_count; + int success_count; //success upload count + int fd; //file description + int64_t time_used; //unit: ms + char *file_buff; //file content +} TestFileInfo; + +#ifdef DEBUG //for debug + +static TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K", 50000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {50 * 1024, "50K", 10000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {200 * 1024, "200K", 5000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {1 * 1024 * 1024, "1M", 500 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {10 * 1024 * 1024, "10M", 50 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {100 * 1024 * 1024, "100M",10 / PROCESS_COUNT, 0, 0, -1, 0, NULL} +}; + +#else + +static TestFileInfo files[FILE_TYPE_COUNT] = { + {5 * 1024, "5K", 1000000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {50 * 1024, "50K", 2000000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {200 * 1024, "200K", 1000000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {1 * 1024 * 1024, "1M", 200000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {10 * 1024 * 1024, "10M", 20000 / PROCESS_COUNT, 0, 0, -1, 0, NULL}, + {100 * 1024 * 1024, "100M", 1000 / PROCESS_COUNT, 0, 0, -1, 0, NULL} +}; + +#endif + +static StorageStat storages[MAX_STORAGE_COUNT]; +static int storage_count = 0; +static time_t start_time; +static int total_count = 0; +static int success_count = 0; +static FILE *fpSuccess = NULL; +static FILE *fpFail = NULL; + +static int proccess_index; +static int load_file_contents(); +static int test_init(); +static int save_stats_by_overall(); +static int save_stats_by_file_type(); +static int save_stats_by_storage_ip(); +static int add_to_storage_stat(const char *storage_ip, const int result, const int time_used); + +int main(int argc, char **argv) +{ + int result; + int upload_count; + int rand_num; + int file_index; + char *conf_filename; + char file_id[128]; + char storage_ip[IP_ADDRESS_SIZE]; + int count_sums[FILE_TYPE_COUNT]; + int i; + struct timeval tv_start; + struct timeval tv_end; + int time_used; + + if (argc < 2) + { + printf("Usage: %s [config_filename]\n", argv[0]); + return EINVAL; + } + + log_init(); + proccess_index = atoi(argv[1]); + if (proccess_index < 0 || proccess_index >= PROCESS_COUNT) + { + printf("Invalid proccess index: %d\n", proccess_index); + return EINVAL; + } + + if (argc >= 3) + { + conf_filename = argv[2]; + } + else + { + conf_filename = "/etc/fdfs/client.conf"; + } + + if ((result = load_file_contents()) != 0) + { + return result; + } + + if ((result=test_init()) != 0) + { + return result; + } + + if ((result=dfs_init(proccess_index, conf_filename)) != 0) + { + return result; + } + +#ifndef WIN32 + if (daemon(1, 1) != 0) + { + return errno != 0 ? errno : EFAULT; + } +#endif + + memset(&storages, 0, sizeof(storages)); + upload_count = 0; + for (i=0; i= files[file_index].count) + { + continue; + } + + files[file_index].upload_count++; + total_count++; + + gettimeofday(&tv_start, NULL); + *storage_ip = '\0'; + + result = upload_file(files[file_index].file_buff, files[file_index].bytes, file_id, storage_ip); + gettimeofday(&tv_end, NULL); + time_used = TIME_SUB_MS(tv_end, tv_start); + files[file_index].time_used += time_used; + + add_to_storage_stat(storage_ip, result, time_used); + if (result == 0) //success + { + success_count++; + files[file_index].success_count++; + + fprintf(fpSuccess, "%d %d %s %s %d\n", + (int)tv_end.tv_sec, files[file_index].bytes, + file_id, storage_ip, time_used); + } + else //fail + { + fprintf(fpFail, "%d %d %d %d\n", (int)tv_end.tv_sec, + files[file_index].bytes, result, time_used); + fflush(fpFail); + } + + if (total_count % 100 == 0) + { + if ((result=save_stats_by_overall()) != 0) + { + break; + } + if ((result=save_stats_by_file_type()) != 0) + { + break; + } + + if ((result=save_stats_by_storage_ip()) != 0) + { + break; + } + } + + } + + save_stats_by_overall(); + save_stats_by_file_type(); + save_stats_by_storage_ip(); + + fclose(fpSuccess); + fclose(fpFail); + + dfs_destroy(); + + printf("proccess %d, time used: %ds\n", proccess_index, (int)(time(NULL) - start_time)); + return result; +} + +static int save_stats_by_file_type() +{ + int k; + char filename[64]; + FILE *fp; + + sprintf(filename, "%s.%d", STAT_FILENAME_BY_FILE_TYPE, proccess_index); + if ((fp=fopen(filename, "wb")) == NULL) + { + printf("open file %s fail, errno: %d, error info: %s\n", + filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EPERM; + } + + fprintf(fp, "#file_type total_count success_count time_used(ms)\n"); + for (k=0; kip_addr) == 0) + { + break; + } + } + + if (pStorage == pEnd) //not found + { + if (storage_count >= MAX_STORAGE_COUNT) + { + printf("storage_count %d >= %d\n", storage_count, MAX_STORAGE_COUNT); + return ENOSPC; + } + + strcpy(pStorage->ip_addr, storage_ip); + storage_count++; + } + + pStorage->time_used += time_used; + pStorage->total_count++; + if (result == 0) + { + pStorage->success_count++; + } + + return 0; +} + +static int load_file_contents() +{ + int i; + //int result; + int64_t file_size; + + for (i=0; i +#include +#include +#include "logger.h" +#include "sockopt.h" +#include "shared_func.h" +#include "tracker_proto.h" +#include "fdfs_global.h" +#include "fdfs_shared_func.h" + +FDFSStorageIdInfo *g_storage_ids_by_ip = NULL; //sorted by group name and storage IP +FDFSStorageIdInfo **g_storage_ids_by_id = NULL; //sorted by storage ID +int g_storage_id_count = 0; + +int fdfs_get_tracker_leader_index_ex(TrackerServerGroup *pServerGroup, \ + const char *leaderIp, const int leaderPort) +{ + ConnectionInfo *pServer; + ConnectionInfo *pEnd; + + if (pServerGroup->server_count == 0) + { + return -1; + } + + pEnd = pServerGroup->servers + pServerGroup->server_count; + for (pServer=pServerGroup->servers; pServerip_addr, leaderIp) == 0 && \ + pServer->port == leaderPort) + { + return pServer - pServerGroup->servers; + } + } + + return -1; +} + +int fdfs_parse_storage_reserved_space(IniContext *pIniContext, \ + FDFSStorageReservedSpace *pStorageReservedSpace) +{ + int result; + int len; + char *pReservedSpaceStr; + int64_t storage_reserved; + + pReservedSpaceStr = iniGetStrValue(NULL, \ + "reserved_storage_space", pIniContext); + if (pReservedSpaceStr == NULL) + { + pStorageReservedSpace->flag = \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB; + pStorageReservedSpace->rs.mb = FDFS_DEF_STORAGE_RESERVED_MB; + return 0; + } + if (*pReservedSpaceStr == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "item \"reserved_storage_space\" is empty!", \ + __LINE__); + return EINVAL; + } + + len = strlen(pReservedSpaceStr); + if (*(pReservedSpaceStr + len - 1) == '%') + { + char *endptr; + pStorageReservedSpace->flag = \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_RATIO; + endptr = NULL; + *(pReservedSpaceStr + len - 1) = '\0'; + pStorageReservedSpace->rs.ratio = \ + strtod(pReservedSpaceStr, &endptr); + if (endptr != NULL && *endptr != '\0') + { + logError("file: "__FILE__", line: %d, " \ + "item \"reserved_storage_space\": %s%%"\ + " is invalid!", __LINE__, \ + pReservedSpaceStr); + return EINVAL; + } + + if (pStorageReservedSpace->rs.ratio <= 0.00 || \ + pStorageReservedSpace->rs.ratio >= 100.00) + { + logError("file: "__FILE__", line: %d, " \ + "item \"reserved_storage_space\": %s%%"\ + " is invalid!", __LINE__, \ + pReservedSpaceStr); + return EINVAL; + } + + pStorageReservedSpace->rs.ratio /= 100.00; + return 0; + } + + if ((result=parse_bytes(pReservedSpaceStr, 1, &storage_reserved)) != 0) + { + return result; + } + + pStorageReservedSpace->flag = TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB; + pStorageReservedSpace->rs.mb = storage_reserved / FDFS_ONE_MB; + return 0; +} + +const char *fdfs_storage_reserved_space_to_string(FDFSStorageReservedSpace \ + *pStorageReservedSpace, char *buff) +{ + if (pStorageReservedSpace->flag == \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB) + { + sprintf(buff, "%d MB", pStorageReservedSpace->rs.mb); + } + else + { + sprintf(buff, "%.2f%%", 100.00 * \ + pStorageReservedSpace->rs.ratio); + } + + return buff; +} + +const char *fdfs_storage_reserved_space_to_string_ex(const bool flag, \ + const int space_mb, const int total_mb, const double space_ratio, \ + char *buff) +{ + if (flag == TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB) + { + sprintf(buff, "%d MB", space_mb); + } + else + { + sprintf(buff, "%d MB(%.2f%%)", (int)(total_mb * space_ratio), \ + 100.00 * space_ratio); + } + + return buff; +} + +int fdfs_get_storage_reserved_space_mb(const int total_mb, \ + FDFSStorageReservedSpace *pStorageReservedSpace) +{ + if (pStorageReservedSpace->flag == \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB) + { + return pStorageReservedSpace->rs.mb; + } + else + { + return (int)(total_mb * pStorageReservedSpace->rs.ratio); + } +} + +bool fdfs_check_reserved_space(FDFSGroupInfo *pGroup, \ + FDFSStorageReservedSpace *pStorageReservedSpace) +{ + if (pStorageReservedSpace->flag == \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB) + { + return pGroup->free_mb > pStorageReservedSpace->rs.mb; + } + else + { + if (pGroup->total_mb == 0) + { + return false; + } + + /* + logInfo("storage=%.4f, rs.ratio=%.4f", + ((double)pGroup->free_mb / (double)pGroup->total_mb), + pStorageReservedSpace->rs.ratio); + */ + + return ((double)pGroup->free_mb / (double)pGroup->total_mb) > \ + pStorageReservedSpace->rs.ratio; + } +} + +bool fdfs_check_reserved_space_trunk(FDFSGroupInfo *pGroup, \ + FDFSStorageReservedSpace *pStorageReservedSpace) +{ + if (pStorageReservedSpace->flag == \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB) + { + return (pGroup->free_mb + pGroup->trunk_free_mb > + pStorageReservedSpace->rs.mb); + } + else + { + if (pGroup->total_mb == 0) + { + return false; + } + + /* + logInfo("storage trunk=%.4f, rs.ratio=%.4f", + ((double)(pGroup->free_mb + pGroup->trunk_free_mb) / \ + (double)pGroup->total_mb), pStorageReservedSpace->rs.ratio); + */ + + return ((double)(pGroup->free_mb + pGroup->trunk_free_mb) / \ + (double)pGroup->total_mb) > pStorageReservedSpace->rs.ratio; + } +} + +bool fdfs_check_reserved_space_path(const int64_t total_mb, \ + const int64_t free_mb, const int avg_mb, \ + FDFSStorageReservedSpace *pStorageReservedSpace) +{ + if (pStorageReservedSpace->flag == \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB) + { + return free_mb > avg_mb; + } + else + { + if (total_mb == 0) + { + return false; + } + + /* + logInfo("storage path, free_mb="INT64_PRINTF_FORMAT \ + ", total_mb="INT64_PRINTF_FORMAT", " \ + "real ratio=%.4f, rs.ratio=%.4f", \ + free_mb, total_mb, ((double)free_mb / total_mb), \ + pStorageReservedSpace->rs.ratio); + */ + + return ((double)free_mb / (double)total_mb) > \ + pStorageReservedSpace->rs.ratio; + } +} + +bool fdfs_is_server_id_valid(const char *id) +{ + long n; + char *endptr; + char buff[FDFS_STORAGE_ID_MAX_SIZE]; + + if (*id == '\0') + { + return false; + } + + endptr = NULL; + n = strtol(id, &endptr, 10); + if (endptr != NULL && *endptr != '\0') + { + return false; + } + + if (n <= 0 || n > FDFS_MAX_SERVER_ID) + { + return false; + } + + snprintf(buff, sizeof(buff), "%ld", n); + return strcmp(buff, id) == 0; +} + +int fdfs_get_server_id_type(const int id) +{ + if (id > 0 && id <= FDFS_MAX_SERVER_ID) + { + return FDFS_ID_TYPE_SERVER_ID; + } + else + { + return FDFS_ID_TYPE_IP_ADDRESS; + } +} + +static int fdfs_cmp_group_name_and_ip(const void *p1, const void *p2) +{ + int result; + result = strcmp(((FDFSStorageIdInfo *)p1)->group_name, + ((FDFSStorageIdInfo *)p2)->group_name); + if (result != 0) + { + return result; + } + + return strcmp(((FDFSStorageIdInfo *)p1)->ip_addr, \ + ((FDFSStorageIdInfo *)p2)->ip_addr); +} + +static int fdfs_cmp_server_id(const void *p1, const void *p2) +{ + return strcmp((*((FDFSStorageIdInfo **)p1))->id, \ + (*((FDFSStorageIdInfo **)p2))->id); +} + +FDFSStorageIdInfo *fdfs_get_storage_id_by_ip(const char *group_name, \ + const char *pIpAddr) +{ + FDFSStorageIdInfo target; + memset(&target, 0, sizeof(FDFSStorageIdInfo)); + snprintf(target.group_name, sizeof(target.group_name), "%s", group_name); + snprintf(target.ip_addr, sizeof(target.ip_addr), "%s", pIpAddr); + return (FDFSStorageIdInfo *)bsearch(&target, g_storage_ids_by_ip, \ + g_storage_id_count, sizeof(FDFSStorageIdInfo), \ + fdfs_cmp_group_name_and_ip); +} + +FDFSStorageIdInfo *fdfs_get_storage_by_id(const char *id) +{ + FDFSStorageIdInfo target; + FDFSStorageIdInfo *pTarget; + FDFSStorageIdInfo **ppFound; + + memset(&target, 0, sizeof(FDFSStorageIdInfo)); + snprintf(target.id, sizeof(target.id), "%s", id); + pTarget = ⌖ + ppFound = (FDFSStorageIdInfo **)bsearch(&pTarget, g_storage_ids_by_id, \ + g_storage_id_count, sizeof(FDFSStorageIdInfo *), \ + fdfs_cmp_server_id); + if (ppFound == NULL) + { + return NULL; + } + else + { + return *ppFound; + } +} + +int fdfs_check_storage_id(const char *group_name, const char *id) +{ + FDFSStorageIdInfo *pFound; + + pFound = fdfs_get_storage_by_id(id); + if (pFound == NULL) + { + return ENOENT; + } + + return strcmp(pFound->group_name, group_name) == 0 ? 0 : EINVAL; +} + +int fdfs_load_storage_ids(char *content, const char *pStorageIdsFilename) +{ + char **lines; + char *line; + char *id; + char *group_name; + char *pIpAddr; + FDFSStorageIdInfo *pStorageIdInfo; + FDFSStorageIdInfo **ppStorageIdInfo; + FDFSStorageIdInfo **ppStorageIdEnd; + int alloc_bytes; + int result; + int line_count; + int i; + + lines = split(content, '\n', 0, &line_count); + if (lines == NULL) + { + return ENOMEM; + } + + result = 0; + do + { + g_storage_id_count = 0; + for (i=0; iip_addr, \ + sizeof(pStorageIdInfo->ip_addr)) == INADDR_NONE) + { + logError("file: "__FILE__", line: %d, " \ + "invalid host name: %s", \ + __LINE__, pIpAddr); + result = EINVAL; + break; + } + + if (!fdfs_is_server_id_valid(id)) + { + logError("file: "__FILE__", line: %d, " \ + "invalid server id: \"%s\", " \ + "which must be a none zero start " \ + "integer, such as 100001", __LINE__, id); + result = EINVAL; + break; + } + + snprintf(pStorageIdInfo->id, \ + sizeof(pStorageIdInfo->id), "%s", id); + snprintf(pStorageIdInfo->group_name, \ + sizeof(pStorageIdInfo->group_name), \ + "%s", group_name); + pStorageIdInfo++; + } + } while (0); + + freeSplit(lines); + if (result != 0) + { + return result; + } + + logDebug("file: "__FILE__", line: %d, " \ + "g_storage_id_count: %d", __LINE__, g_storage_id_count); + pStorageIdInfo = g_storage_ids_by_ip; + for (i=0; iid, \ + pStorageIdInfo->group_name, + pStorageIdInfo->ip_addr); + + pStorageIdInfo++; + } + + ppStorageIdEnd = g_storage_ids_by_id + g_storage_id_count; + pStorageIdInfo = g_storage_ids_by_ip; + for (ppStorageIdInfo=g_storage_ids_by_id; ppStorageIdInfo < \ + ppStorageIdEnd; ppStorageIdInfo++) + { + *ppStorageIdInfo = pStorageIdInfo++; + } + + qsort(g_storage_ids_by_ip, g_storage_id_count, \ + sizeof(FDFSStorageIdInfo), fdfs_cmp_group_name_and_ip); + qsort(g_storage_ids_by_id, g_storage_id_count, \ + sizeof(FDFSStorageIdInfo *), fdfs_cmp_server_id); + return result; +} + +int fdfs_get_storage_ids_from_tracker_server(ConnectionInfo *pTrackerServer) +{ +#define MAX_REQUEST_LOOP 32 + TrackerHeader *pHeader; + ConnectionInfo *conn; + char out_buff[sizeof(TrackerHeader) + sizeof(int)]; + char *p; + char *response; + struct data_info { + char *buffer; //for free + char *content; + int length; + } data_list[MAX_REQUEST_LOOP]; + int list_count; + int total_count; + int current_count; + int result; + int i; + int start_index; + int64_t in_bytes; + + if ((conn=tracker_connect_server(pTrackerServer, &result)) == NULL) + { + return result; + } + + memset(data_list, 0, sizeof(data_list)); + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + p = out_buff + sizeof(TrackerHeader); + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_FETCH_STORAGE_IDS; + long2buff(sizeof(int), pHeader->pkg_len); + + start_index = 0; + list_count = 0; + result = 0; + while (1) + { + int2buff(start_index, p); + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + } + else + { + response = NULL; + result = fdfs_recv_response(conn, \ + &response, 0, &in_bytes); + } + + if (result != 0) + { + break; + } + + if (in_bytes < 2 * sizeof(int)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, recv data length: %d "\ + "is invalid", __LINE__, + pTrackerServer->ip_addr, \ + pTrackerServer->port, (int)in_bytes); + result = EINVAL; + break; + } + + total_count = buff2int(response); + current_count = buff2int(response + sizeof(int)); + if (total_count <= start_index) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, total storage " \ + "count: %d is invalid, which <= start " \ + "index: %d", __LINE__, pTrackerServer->ip_addr,\ + pTrackerServer->port, total_count, start_index); + result = EINVAL; + break; + } + + if (current_count <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, current storage " \ + "count: %d is invalid, which <= 0", \ + __LINE__, pTrackerServer->ip_addr,\ + pTrackerServer->port, current_count); + result = EINVAL; + break; + } + + data_list[list_count].buffer = response; + data_list[list_count].content = response + 2 * sizeof(int); + data_list[list_count].length = in_bytes - 2 * sizeof(int); + list_count++; + + /* + //logInfo("list_count: %d, total_count: %d, current_count: %d", + list_count, total_count, current_count); + */ + + start_index += current_count; + if (start_index >= total_count) + { + break; + } + + if (list_count == MAX_REQUEST_LOOP) + { + logError("file: "__FILE__", line: %d, " \ + "response data from tracker " \ + "server %s:%d is too large", \ + __LINE__, pTrackerServer->ip_addr,\ + pTrackerServer->port); + result = ENOSPC; + break; + } + } + + tracker_disconnect_server_ex(conn, result != 0); + + if (result == 0) + { + do + { + int total_length; + char *content; + + total_length = 0; + for (i=0; iservers + pTrackerGroup->server_count; + + leader_index = pTrackerGroup->leader_index; + if (leader_index >= 0) + { + pServerStart = pTrackerGroup->servers + leader_index; + } + else + { + pServerStart = pTrackerGroup->servers; + } + + result = ENOENT; + for (i=0; i<5; i++) + { + for (pGServer=pServerStart; pGServersock = -1; + result = fdfs_get_storage_ids_from_tracker_server(pTServer); + if (result == 0) + { + return result; + } + } + + if (pServerStart != pTrackerGroup->servers) + { + pServerStart = pTrackerGroup->servers; + } + sleep(1); + } + + return result; +} + +int fdfs_load_storage_ids_from_file(const char *config_filename, \ + IniContext *pItemContext) +{ + char *pStorageIdsFilename; + char *content; + int64_t file_size; + int result; + + pStorageIdsFilename = iniGetStrValue(NULL, "storage_ids_filename", \ + pItemContext); + if (pStorageIdsFilename == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" must have item " \ + "\"storage_ids_filename\"!", __LINE__, config_filename); + return ENOENT; + } + + if (*pStorageIdsFilename == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", storage_ids_filename is emtpy!", \ + __LINE__, config_filename); + return EINVAL; + } + + if (*pStorageIdsFilename == '/') //absolute path + { + result = getFileContent(pStorageIdsFilename, \ + &content, &file_size); + } + else + { + const char *lastSlash = strrchr(config_filename, '/'); + if (lastSlash == NULL) + { + result = getFileContent(pStorageIdsFilename, \ + &content, &file_size); + } + else + { + char filepath[MAX_PATH_SIZE]; + char full_filename[MAX_PATH_SIZE]; + int len; + + len = lastSlash - config_filename; + if (len >= sizeof(filepath)) + { + logError("file: "__FILE__", line: %d, " \ + "conf filename: \"%s\" is too long!", \ + __LINE__, config_filename); + return ENOSPC; + } + memcpy(filepath, config_filename, len); + *(filepath + len) = '\0'; + snprintf(full_filename, sizeof(full_filename), \ + "%s/%s", filepath, pStorageIdsFilename); + result = getFileContent(full_filename, \ + &content, &file_size); + } + } + if (result != 0) + { + return result; + } + + result = fdfs_load_storage_ids(content, pStorageIdsFilename); + free(content); + return result; +} + +int fdfs_connection_pool_init(const char *config_filename, \ + IniContext *pItemContext) +{ + g_use_connection_pool = iniGetBoolValue(NULL, "use_connection_pool", \ + pItemContext, false); + if (!g_use_connection_pool) + { + return 0; + } + + g_connection_pool_max_idle_time = iniGetIntValue(NULL, \ + "connection_pool_max_idle_time", \ + pItemContext, 3600); + if (g_connection_pool_max_idle_time <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "connection_pool_max_idle_time: %d of conf " \ + "filename: \"%s\" is invalid!", __LINE__, \ + g_connection_pool_max_idle_time, config_filename); + return EINVAL; + } + + return conn_pool_init(&g_connection_pool, g_fdfs_connect_timeout, \ + 0, g_connection_pool_max_idle_time); +} + +void fdfs_connection_pool_destroy() +{ + conn_pool_destroy(&g_connection_pool); +} + diff --git a/tracker/fdfs_shared_func.h b/tracker/fdfs_shared_func.h new file mode 100644 index 0000000..172831f --- /dev/null +++ b/tracker/fdfs_shared_func.h @@ -0,0 +1,82 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//fdfs_shared_func.h + +#ifndef _FDFS_SHARED_FUNC_H +#define _FDFS_SHARED_FUNC_H + +#include "common_define.h" +#include "ini_file_reader.h" +#include "tracker_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern FDFSStorageIdInfo *g_storage_ids_by_ip; //sorted by group name and storage IP +extern FDFSStorageIdInfo **g_storage_ids_by_id; //sorted by storage ID +extern int g_storage_id_count; //storage id count + +int fdfs_get_tracker_leader_index_ex(TrackerServerGroup *pServerGroup, \ + const char *leaderIp, const int leaderPort); + +int fdfs_parse_storage_reserved_space(IniContext *pIniContext, \ + FDFSStorageReservedSpace *pStorageReservedSpace); + +const char *fdfs_storage_reserved_space_to_string(FDFSStorageReservedSpace \ + *pStorageReservedSpace, char *buff); + +const char *fdfs_storage_reserved_space_to_string_ex(const bool flag, \ + const int space_mb, const int total_mb, const double space_ratio, \ + char *buff); + +int fdfs_get_storage_reserved_space_mb(const int total_mb, \ + FDFSStorageReservedSpace *pStorageReservedSpace); + +bool fdfs_check_reserved_space(FDFSGroupInfo *pGroup, \ + FDFSStorageReservedSpace *pStorageReservedSpace); + +bool fdfs_check_reserved_space_trunk(FDFSGroupInfo *pGroup, \ + FDFSStorageReservedSpace *pStorageReservedSpace); + +bool fdfs_check_reserved_space_path(const int64_t total_mb, \ + const int64_t free_mb, const int avg_mb, \ + FDFSStorageReservedSpace *pStorageReservedSpace); + +bool fdfs_is_server_id_valid(const char *id); + +int fdfs_get_server_id_type(const int id); + +int fdfs_load_storage_ids(char *content, const char *pStorageIdsFilename); + +FDFSStorageIdInfo *fdfs_get_storage_by_id(const char *id); + +FDFSStorageIdInfo *fdfs_get_storage_id_by_ip(const char *group_name, \ + const char *pIpAddr); + +int fdfs_check_storage_id(const char *group_name, const char *id); + +int fdfs_get_storage_ids_from_tracker_server(ConnectionInfo *pTrackerServer); + +int fdfs_get_storage_ids_from_tracker_group(TrackerServerGroup *pTrackerGroup); + +int fdfs_load_storage_ids_from_file(const char *config_filename, \ + IniContext *pItemContext); + +int fdfs_connection_pool_init(const char *config_filename, \ + IniContext *pItemContext); + +void fdfs_connection_pool_destroy(); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/tracker/fdfs_trackerd.c b/tracker/fdfs_trackerd.c new file mode 100644 index 0000000..07dfed2 --- /dev/null +++ b/tracker/fdfs_trackerd.c @@ -0,0 +1,547 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "pthread_func.h" +#include "process_ctrl.h" +#include "logger.h" +#include "fdfs_global.h" +#include "base64.h" +#include "sockopt.h" +#include "sched_thread.h" +#include "tracker_types.h" +#include "tracker_mem.h" +#include "tracker_service.h" +#include "tracker_global.h" +#include "tracker_proto.h" +#include "tracker_func.h" +#include "tracker_status.h" +#include "tracker_relationship.h" + +#ifdef WITH_HTTPD +#include "tracker_httpd.h" +#include "tracker_http_check.h" +#endif + +#if defined(DEBUG_FLAG) + +/* +#if defined(OS_LINUX) +#include "linux_stack_trace.h" +static bool bSegmentFault = false; +#endif +*/ + +#include "tracker_dump.h" +#endif + +static bool bTerminateFlag = false; +static bool bAcceptEndFlag = false; + +static char bind_addr[IP_ADDRESS_SIZE]; + +static void sigQuitHandler(int sig); +static void sigHupHandler(int sig); +static void sigUsrHandler(int sig); +static void sigAlarmHandler(int sig); + +#if defined(DEBUG_FLAG) +/* +#if defined(OS_LINUX) +static void sigSegvHandler(int signum, siginfo_t *info, void *ptr); +#endif +*/ + +static void sigDumpHandler(int sig); +#endif + +#define SCHEDULE_ENTRIES_COUNT 4 + +static void usage(const char *program) +{ + fprintf(stderr, "Usage: %s [start | stop | restart]\n", + program); +} + +int main(int argc, char *argv[]) +{ + char *conf_filename; + int result; + int wait_count; + int sock; + pthread_t schedule_tid; + struct sigaction act; + ScheduleEntry scheduleEntries[SCHEDULE_ENTRIES_COUNT]; + ScheduleArray scheduleArray; + char pidFilename[MAX_PATH_SIZE]; + bool stop; + + if (argc < 2) + { + usage(argv[0]); + return 1; + } + + g_current_time = time(NULL); + g_up_time = g_current_time; + srand(g_up_time); + + log_init(); + + conf_filename = argv[1]; + if ((result=get_base_path_from_conf_file(conf_filename, + g_fdfs_base_path, sizeof(g_fdfs_base_path))) != 0) + { + log_destroy(); + return result; + } + + snprintf(pidFilename, sizeof(pidFilename), + "%s/data/fdfs_trackerd.pid", g_fdfs_base_path); + if ((result=process_action(pidFilename, argv[2], &stop)) != 0) + { + if (result == EINVAL) + { + usage(argv[0]); + } + log_destroy(); + return result; + } + if (stop) + { + log_destroy(); + return 0; + } + +#if defined(DEBUG_FLAG) && defined(OS_LINUX) + if (getExeAbsoluteFilename(argv[0], g_exe_name, \ + sizeof(g_exe_name)) == NULL) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return errno != 0 ? errno : ENOENT; + } +#endif + + memset(bind_addr, 0, sizeof(bind_addr)); + if ((result=tracker_load_from_conf_file(conf_filename, \ + bind_addr, sizeof(bind_addr))) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + if ((result=tracker_load_status_from_file(&g_tracker_last_status)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + base64_init_ex(&g_base64_context, 0, '-', '_', '.'); + if ((result=set_rand_seed()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "set_rand_seed fail, program exit!", __LINE__); + return result; + } + + if ((result=tracker_mem_init()) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + sock = socketServer(bind_addr, g_server_port, &result); + if (sock < 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + if ((result=tcpsetserveropt(sock, g_fdfs_network_timeout)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + daemon_init(true); + umask(0); + + if ((result=write_to_pid_file(pidFilename)) != 0) + { + log_destroy(); + return result; + } + + if (dup2(g_log_context.log_fd, STDOUT_FILENO) < 0 || \ + dup2(g_log_context.log_fd, STDERR_FILENO) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call dup2 fail, errno: %d, error info: %s, " \ + "program exit!", __LINE__, errno, STRERROR(errno)); + g_continue_flag = false; + return errno; + } + + if ((result=tracker_service_init()) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + + act.sa_handler = sigUsrHandler; + if(sigaction(SIGUSR1, &act, NULL) < 0 || \ + sigaction(SIGUSR2, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + + act.sa_handler = sigHupHandler; + if(sigaction(SIGHUP, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + + act.sa_handler = SIG_IGN; + if(sigaction(SIGPIPE, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + + act.sa_handler = sigQuitHandler; + if(sigaction(SIGINT, &act, NULL) < 0 || \ + sigaction(SIGTERM, &act, NULL) < 0 || \ + sigaction(SIGQUIT, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } + +#if defined(DEBUG_FLAG) +/* +#if defined(OS_LINUX) + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + act.sa_sigaction = sigSegvHandler; + act.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &act, NULL) < 0 || \ + sigaction(SIGABRT, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } +#endif +*/ + + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + act.sa_handler = sigDumpHandler; + if(sigaction(SIGUSR1, &act, NULL) < 0 || \ + sigaction(SIGUSR2, &act, NULL) < 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "call sigaction fail, errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + logCrit("exit abnormally!\n"); + return errno; + } +#endif + +#ifdef WITH_HTTPD + if (!g_http_params.disabled) + { + if ((result=tracker_httpd_start(bind_addr)) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "tracker_httpd_start fail, program exit!", \ + __LINE__); + return result; + } + + } + + if ((result=tracker_http_check_start()) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "tracker_http_check_start fail, " \ + "program exit!", __LINE__); + return result; + } +#endif + + if ((result=set_run_by(g_run_by_group, g_run_by_user)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + scheduleArray.entries = scheduleEntries; + memset(scheduleEntries, 0, sizeof(scheduleEntries)); + scheduleEntries[0].id = 1; + scheduleEntries[0].time_base.hour = TIME_NONE; + scheduleEntries[0].time_base.minute = TIME_NONE; + scheduleEntries[0].interval = g_sync_log_buff_interval; + scheduleEntries[0].task_func = log_sync_func; + scheduleEntries[0].func_args = &g_log_context; + + scheduleEntries[1].id = 2; + scheduleEntries[1].time_base.hour = TIME_NONE; + scheduleEntries[1].time_base.minute = TIME_NONE; + scheduleEntries[1].interval = g_check_active_interval; + scheduleEntries[1].task_func = tracker_mem_check_alive; + scheduleEntries[1].func_args = NULL; + + scheduleEntries[2].id = 3; + scheduleEntries[2].time_base.hour = 0; + scheduleEntries[2].time_base.minute = 0; + scheduleEntries[2].interval = TRACKER_SYNC_STATUS_FILE_INTERVAL; + scheduleEntries[2].task_func = tracker_write_status_to_file; + scheduleEntries[2].func_args = NULL; + + scheduleArray.count = 3; + + if (g_rotate_error_log) + { + scheduleEntries[scheduleArray.count].id = 4; + scheduleEntries[scheduleArray.count].time_base = \ + g_error_log_rotate_time; + scheduleEntries[scheduleArray.count].interval = \ + 24 * 3600; + scheduleEntries[scheduleArray.count].task_func = \ + log_notify_rotate; + scheduleEntries[scheduleArray.count].func_args = \ + &g_log_context; + scheduleArray.count++; + } + + if ((result=sched_start(&scheduleArray, &schedule_tid, \ + g_thread_stack_size, (bool * volatile)&g_continue_flag)) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + if ((result=tracker_relationship_init()) != 0) + { + logCrit("exit abnormally!\n"); + log_destroy(); + return result; + } + + log_set_cache(true); + + bTerminateFlag = false; + bAcceptEndFlag = false; + + tracker_accept_loop(sock); + bAcceptEndFlag = true; + if (g_schedule_flag) + { + pthread_kill(schedule_tid, SIGINT); + } + tracker_terminate_threads(); + +#ifdef WITH_HTTPD + if (g_http_check_flag) + { + tracker_http_check_stop(); + } + + while (g_http_check_flag) + { + usleep(50000); + } +#endif + + wait_count = 0; + while ((g_tracker_thread_count != 0) || g_schedule_flag) + { + +/* +#if defined(DEBUG_FLAG) && defined(OS_LINUX) + if (bSegmentFault) + { + sleep(5); + break; + } +#endif +*/ + + usleep(10000); + if (++wait_count > 3000) + { + logWarning("waiting timeout, exit!"); + break; + } + } + + tracker_mem_destroy(); + tracker_service_destroy(); + tracker_relationship_destroy(); + + logInfo("exit normally.\n"); + log_destroy(); + + delete_pid_file(pidFilename); + return 0; +} + +#if defined(DEBUG_FLAG) +/* +#if defined(OS_LINUX) +static void sigSegvHandler(int signum, siginfo_t *info, void *ptr) +{ + bSegmentFault = true; + + if (!bTerminateFlag) + { + set_timer(1, 1, sigAlarmHandler); + + bTerminateFlag = true; + g_continue_flag = false; + + logCrit("file: "__FILE__", line: %d, " \ + "catch signal %d, program exiting...", \ + __LINE__, signum); + + signal_stack_trace_print(signum, info, ptr); + } +} +#endif +*/ + +static void sigDumpHandler(int sig) +{ + static bool bDumpFlag = false; + char filename[256]; + + if (bDumpFlag) + { + return; + } + + bDumpFlag = true; + + snprintf(filename, sizeof(filename), + "%s/logs/tracker_dump.log", g_fdfs_base_path); + fdfs_dump_tracker_global_vars_to_file(filename); + + bDumpFlag = false; +} + +#endif + +static void sigQuitHandler(int sig) +{ + if (!bTerminateFlag) + { + set_timer(1, 1, sigAlarmHandler); + + bTerminateFlag = true; + g_continue_flag = false; + logCrit("file: "__FILE__", line: %d, " \ + "catch signal %d, program exiting...", \ + __LINE__, sig); + } +} + +static void sigHupHandler(int sig) +{ + if (g_rotate_error_log) + { + g_log_context.rotate_immediately = true; + } + + logInfo("file: "__FILE__", line: %d, " \ + "catch signal %d, rotate log", __LINE__, sig); +} + +static void sigAlarmHandler(int sig) +{ + ConnectionInfo server; + + if (bAcceptEndFlag) + { + return; + } + + logDebug("file: "__FILE__", line: %d, " \ + "signal server to quit...", __LINE__); + + if (*bind_addr != '\0') + { + strcpy(server.ip_addr, bind_addr); + } + else + { + strcpy(server.ip_addr, "127.0.0.1"); + } + server.port = g_server_port; + server.sock = -1; + + if (conn_pool_connect_server(&server, g_fdfs_connect_timeout) != 0) + { + return; + } + + fdfs_quit(&server); + conn_pool_disconnect_server(&server); + + logDebug("file: "__FILE__", line: %d, " \ + "signal server to quit done", __LINE__); +} + +static void sigUsrHandler(int sig) +{ + logInfo("file: "__FILE__", line: %d, " \ + "catch signal %d, ignore it", __LINE__, sig); +} + diff --git a/tracker/tracker_dump.c b/tracker/tracker_dump.c new file mode 100644 index 0000000..109ccce --- /dev/null +++ b/tracker/tracker_dump.c @@ -0,0 +1,546 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include "tracker_dump.h" +#include "shared_func.h" +#include "sched_thread.h" +#include "logger.h" +#include "hash.h" +#include "connection_pool.h" +#include "fdfs_global.h" +#include "tracker_global.h" +#include "tracker_mem.h" +#include "tracker_service.h" +#include "tracker_relationship.h" +#include "fdfs_shared_func.h" + +static int fdfs_dump_storage_stat(FDFSStorageDetail *pServer, + char *buff, const int buffSize); + +static int fdfs_dump_group_stat(FDFSGroupInfo *pGroup, char *buff, const int buffSize) +{ + char szLastSourceUpdate[32]; + char szLastSyncUpdate[32]; + char szSyncedTimestamp[32]; + int total_len; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + int i; + int j; + + total_len = snprintf(buff, buffSize, + "group_name=%s\n" + "total_mb="INT64_PRINTF_FORMAT"\n" + "free_mb="INT64_PRINTF_FORMAT"\n" + "alloc_size=%d\n" + "server count=%d\n" + "active server count=%d\n" + "storage_port=%d\n" + "storage_http_port=%d\n" + "current_read_server=%d\n" + "current_write_server=%d\n" + "store_path_count=%d\n" + "subdir_count_per_path=%d\n" + "current_trunk_file_id=%d\n" + "pStoreServer=%s\n" + "pTrunkServer=%s\n" + "last_trunk_server_id=%s\n" + "chg_count=%d\n" + "trunk_chg_count=%d\n" + "last_source_update=%s\n" + "last_sync_update=%s\n", + pGroup->group_name, + pGroup->total_mb, + pGroup->free_mb, + pGroup->alloc_size, + pGroup->count, + pGroup->active_count, + pGroup->storage_port, + pGroup->storage_http_port, + pGroup->current_read_server, + pGroup->current_write_server, + pGroup->store_path_count, + pGroup->subdir_count_per_path, + pGroup->current_trunk_file_id, + pGroup->pStoreServer != NULL ? pGroup->pStoreServer->ip_addr : "", + pGroup->pTrunkServer != NULL ? pGroup->pTrunkServer->ip_addr : "", + pGroup->last_trunk_server_id, + pGroup->chg_count, + pGroup->trunk_chg_count, + formatDatetime(pGroup->last_source_update, + "%Y-%m-%d %H:%M:%S", + szLastSourceUpdate, sizeof(szLastSourceUpdate)), + formatDatetime(pGroup->last_sync_update, + "%Y-%m-%d %H:%M:%S", + szLastSyncUpdate, sizeof(szLastSyncUpdate)) + ); + + total_len += snprintf(buff + total_len, buffSize - total_len, + "total server count=%d\n", pGroup->count); + ppServerEnd = pGroup->all_servers + pGroup->count; + for (ppServer=pGroup->all_servers; ppServerip_addr); + } + + total_len += snprintf(buff + total_len, buffSize - total_len, + "\nactive server count=%d\n", pGroup->active_count); + ppServerEnd = pGroup->active_servers + pGroup->active_count; + for (ppServer=pGroup->active_servers; ppServerip_addr); + } + +#ifdef WITH_HTTPD + total_len += snprintf(buff + total_len, buffSize - total_len, + "\nhttp active server count=%d\n" + "current_http_server=%d\n", + pGroup->http_server_count, + pGroup->current_http_server); + + ppServerEnd = pGroup->http_servers + pGroup->http_server_count; + for (ppServer=pGroup->http_servers; ppServerip_addr); + } +#endif + + ppServerEnd = pGroup->sorted_servers + pGroup->count; + for (ppServer=pGroup->sorted_servers; ppServersorted_servers) + 1); + total_len += fdfs_dump_storage_stat(*ppServer, buff + total_len, + buffSize - total_len); + } + + total_len += snprintf(buff + total_len, buffSize - total_len, + "\nsynced timestamp table:\n"); + for (i=0; icount; i++) + { + for (j=0; jcount; j++) + { + if (i == j) + { + continue; + } + + total_len += snprintf(buff + total_len, buffSize - total_len, + "\t%s => %s: %s\n", + pGroup->all_servers[i]->ip_addr, + pGroup->all_servers[j]->ip_addr, + formatDatetime(pGroup->last_sync_timestamps[i][j], + "%Y-%m-%d %H:%M:%S", + szSyncedTimestamp, + sizeof(szSyncedTimestamp)) + ); + } + } + + total_len += snprintf(buff + total_len, buffSize - total_len, + "\n\n"); + return total_len; +} + +static int fdfs_dump_storage_stat(FDFSStorageDetail *pServer, + char *buff, const int buffSize) +{ + char szJoinTime[32]; + char szUpTime[32]; + char szLastHeartBeatTime[32]; + char szSrcUpdTime[32]; + char szSyncUpdTime[32]; + char szSyncedTimestamp[32]; + char szSyncUntilTimestamp[32]; + int i; + int total_len; + + total_len = snprintf(buff, buffSize, + "ip_addr=%s\n" + "version=%s\n" + "status=%d\n" + "domain_name=%s\n" + "sync_src_server=%s\n" + "sync_until_timestamp=%s\n" + "join_time=%s\n" + "up_time=%s\n" + "total_mb="INT64_PRINTF_FORMAT" MB\n" + "free_mb="INT64_PRINTF_FORMAT" MB\n" + "changelog_offset="INT64_PRINTF_FORMAT"\n" + "store_path_count=%d\n" + "storage_port=%d\n" + "storage_http_port=%d\n" + "subdir_count_per_path=%d\n" + "upload_priority=%d\n" + "current_write_path=%d\n" + "chg_count=%d\n" +#ifdef WITH_HTTPD + "http_check_last_errno=%d\n" + "http_check_last_status=%d\n" + "http_check_fail_count=%d\n" + "http_check_error_info=%s\n" +#endif + + "total_upload_count="INT64_PRINTF_FORMAT"\n" + "success_upload_count="INT64_PRINTF_FORMAT"\n" + "total_set_meta_count="INT64_PRINTF_FORMAT"\n" + "success_set_meta_count="INT64_PRINTF_FORMAT"\n" + "total_delete_count="INT64_PRINTF_FORMAT"\n" + "success_delete_count="INT64_PRINTF_FORMAT"\n" + "total_download_count="INT64_PRINTF_FORMAT"\n" + "success_download_count="INT64_PRINTF_FORMAT"\n" + "total_get_meta_count="INT64_PRINTF_FORMAT"\n" + "success_get_meta_count="INT64_PRINTF_FORMAT"\n" + "total_create_link_count="INT64_PRINTF_FORMAT"\n" + "success_create_link_count="INT64_PRINTF_FORMAT"\n" + "total_delete_link_count="INT64_PRINTF_FORMAT"\n" + "success_delete_link_count="INT64_PRINTF_FORMAT"\n" + "last_source_update=%s\n" + "last_sync_update=%s\n" + "last_synced_timestamp=%s\n" + "last_heart_beat_time=%s\n", + pServer->ip_addr, + pServer->version, + pServer->status, + pServer->domain_name, + pServer->psync_src_server != NULL ? + pServer->psync_src_server->ip_addr : "", + formatDatetime(pServer->sync_until_timestamp, + "%Y-%m-%d %H:%M:%S", + szSyncUntilTimestamp, sizeof(szSyncUntilTimestamp)), + formatDatetime(pServer->join_time, + "%Y-%m-%d %H:%M:%S", + szJoinTime, sizeof(szJoinTime)), + formatDatetime(pServer->up_time, + "%Y-%m-%d %H:%M:%S", + szUpTime, sizeof(szUpTime)), + pServer->total_mb, + pServer->free_mb, + pServer->changelog_offset, + pServer->store_path_count, + pServer->storage_port, + pServer->storage_http_port, + pServer->subdir_count_per_path, + pServer->upload_priority, + pServer->current_write_path, + pServer->chg_count, +#ifdef WITH_HTTPD + pServer->http_check_last_errno, + pServer->http_check_last_status, + pServer->http_check_fail_count, + pServer->http_check_error_info, +#endif + pServer->stat.total_upload_count, + pServer->stat.success_upload_count, + pServer->stat.total_set_meta_count, + pServer->stat.success_set_meta_count, + pServer->stat.total_delete_count, + pServer->stat.success_delete_count, + pServer->stat.total_download_count, + pServer->stat.success_download_count, + pServer->stat.total_get_meta_count, + pServer->stat.success_get_meta_count, + pServer->stat.total_create_link_count, + pServer->stat.success_create_link_count, + pServer->stat.total_delete_link_count, + pServer->stat.success_delete_link_count, + formatDatetime(pServer->stat.last_source_update, + "%Y-%m-%d %H:%M:%S", + szSrcUpdTime, sizeof(szSrcUpdTime)), + formatDatetime(pServer->stat.last_sync_update, + "%Y-%m-%d %H:%M:%S", + szSyncUpdTime, sizeof(szSyncUpdTime)), + formatDatetime(pServer->stat.last_synced_timestamp, + "%Y-%m-%d %H:%M:%S", + szSyncedTimestamp, sizeof(szSyncedTimestamp)), + formatDatetime(pServer->stat.last_heart_beat_time, + "%Y-%m-%d %H:%M:%S", + szLastHeartBeatTime, sizeof(szLastHeartBeatTime)) + ); + + for (i=0; istore_path_count; i++) + { + total_len += snprintf(buff + total_len, buffSize - total_len, + "disk %d: total_mb="INT64_PRINTF_FORMAT" MB, " + "free_mb="INT64_PRINTF_FORMAT" MB\n", + i+1, pServer->path_total_mbs[i], + pServer->path_free_mbs[i]); + } + + return total_len; +} + +static int fdfs_dump_global_vars(char *buff, const int buffSize) +{ + int total_len; + char reserved_space_str[32]; + + total_len = snprintf(buff, buffSize, + "g_fdfs_connect_timeout=%ds\n" + "g_fdfs_network_timeout=%ds\n" + "g_fdfs_base_path=%s\n" + "g_fdfs_version=%d.%02d\n" + "g_continue_flag=%d\n" + "g_schedule_flag=%d\n" + "g_server_port=%d\n" + "g_max_connections=%d\n" + "g_tracker_thread_count=%d\n" + "g_sync_log_buff_interval=%ds\n" + "g_check_active_interval=%ds\n" + "g_storage_stat_chg_count=%d\n" + "g_storage_sync_time_chg_count=%d\n" + "g_storage_reserved_space=%s\n" + "g_allow_ip_count=%d\n" + "g_run_by_group=%s\n" + "g_run_by_user=%s\n" + "g_storage_ip_changed_auto_adjust=%d\n" + "g_thread_stack_size=%d\n" + "if_use_trunk_file=%d\n" + "slot_min_size=%d\n" + "slot_max_size=%d MB\n" + "trunk_file_size=%d MB\n" + "g_changelog_fsize="INT64_PRINTF_FORMAT"\n" + "g_storage_sync_file_max_delay=%ds\n" + "g_storage_sync_file_max_time=%ds\n" + "g_up_time=%d\n" + "g_tracker_last_status.up_time=%d\n" + "g_tracker_last_status.last_check_time=%d\n" + "g_if_leader_self=%d\n" + "g_next_leader_index=%d\n" + "g_tracker_leader_chg_count=%d\n" + "g_trunk_server_chg_count=%d\n" + "g_use_connection_pool=%d\n" + "g_connection_pool_max_idle_time=%d\n" + "connection_pool_conn_count=%d\n" + #ifdef WITH_HTTPD + "g_http_params.disabled=%d\n" + "g_http_params.anti_steal_token=%d\n" + "g_http_params.server_port=%d\n" + "g_http_params.content_type_hash item count=%d\n" + "g_http_params.anti_steal_secret_key length=%d\n" + "g_http_params.token_check_fail_buff length=%d\n" + "g_http_params.default_content_type=%s\n" + "g_http_params.token_check_fail_content_type=%s\n" + "g_http_params.token_ttl=%d\n" + "g_http_check_interval=%d\n" + "g_http_check_type=%d\n" + "g_http_check_uri=%s\n" + "g_http_servers_dirty=%d\n" + #endif + #if defined(DEBUG_FLAG) && defined(OS_LINUX) + "g_exe_name=%s\n" + #endif + , g_fdfs_connect_timeout + , g_fdfs_network_timeout + , g_fdfs_base_path + , g_fdfs_version.major, g_fdfs_version.minor + , g_continue_flag + , g_schedule_flag + , g_server_port + , g_max_connections + , g_tracker_thread_count + , g_sync_log_buff_interval + , g_check_active_interval + , g_storage_stat_chg_count + , g_storage_sync_time_chg_count + , fdfs_storage_reserved_space_to_string( \ + &g_storage_reserved_space, reserved_space_str) \ + , g_allow_ip_count + , g_run_by_group + , g_run_by_user + , g_storage_ip_changed_auto_adjust + , g_thread_stack_size + , g_if_use_trunk_file + , g_slot_min_size + , g_slot_max_size / FDFS_ONE_MB + , g_trunk_file_size / FDFS_ONE_MB + , g_changelog_fsize + , g_storage_sync_file_max_delay + , g_storage_sync_file_max_time + , (int)g_up_time + , (int)g_tracker_last_status.up_time + , (int)g_tracker_last_status.last_check_time + , g_if_leader_self + , g_next_leader_index + , g_tracker_leader_chg_count + , g_trunk_server_chg_count + , g_use_connection_pool + , g_connection_pool_max_idle_time + , g_use_connection_pool ? conn_pool_get_connection_count( \ + &g_connection_pool) : 0 + #ifdef WITH_HTTPD + , g_http_params.disabled + , g_http_params.anti_steal_token + , g_http_params.server_port + , hash_count(&(g_http_params.content_type_hash)) + , g_http_params.anti_steal_secret_key.length + , g_http_params.token_check_fail_buff.length + , g_http_params.default_content_type + , g_http_params.token_check_fail_content_type + , g_http_params.token_ttl + , g_http_check_interval + , g_http_check_type + , g_http_check_uri + , g_http_servers_dirty + #endif + + #if defined(DEBUG_FLAG) && defined(OS_LINUX) + , g_exe_name + #endif + ); + + return total_len; +} + +static int fdfs_dump_tracker_servers(char *buff, const int buffSize) +{ + int total_len; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pTrackerEnd; + + total_len = snprintf(buff, buffSize, \ + "g_tracker_servers.server_count=%d, " \ + "g_tracker_servers.leader_index=%d\n", \ + g_tracker_servers.server_count, \ + g_tracker_servers.leader_index); + if (g_tracker_servers.server_count == 0) + { + return total_len; + } + + pTrackerEnd = g_tracker_servers.servers + g_tracker_servers.server_count; + for (pTrackerServer=g_tracker_servers.servers; \ + pTrackerServerip_addr, pTrackerServer->port); + } + + return total_len; +} + +static int fdfs_dump_groups_info(char *buff, const int buffSize) +{ + int total_len; + + total_len = snprintf(buff, buffSize, + "group count=%d\n" + "group alloc_size=%d\n" + "store_lookup=%d\n" + "store_server=%d\n" + "download_server=%d\n" + "store_path=%d\n" + "store_group=%s\n" + "pStoreGroup=%s\n" + "current_write_group=%d\n", + g_groups.count, + g_groups.alloc_size, + g_groups.store_lookup, + g_groups.store_server, + g_groups.download_server, + g_groups.store_path, + g_groups.store_group, + g_groups.pStoreGroup != NULL ? + g_groups.pStoreGroup->group_name : "", + g_groups.current_write_group + ); + + return total_len; +} + +#define WRITE_TO_FILE(fd, buff, len) \ + if (write(fd, buff, len) != len) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "write to file %s fail, errno: %d, error info: %s", \ + __LINE__, filename, errno, STRERROR(errno)); \ + result = errno; \ + break; \ + } + +int fdfs_dump_tracker_global_vars_to_file(const char *filename) +{ + char buff[16 * 1024]; + char szCurrentTime[32]; + int len; + int result; + int fd; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + + fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd < 0) + { + logError("file: "__FILE__", line: %d, " + "open file %s fail, errno: %d, error info: %s", + __LINE__, filename, errno, STRERROR(errno)); + return errno; + } + + do + { + result = 0; + formatDatetime(g_current_time, "%Y-%m-%d %H:%M:%S", + szCurrentTime, sizeof(szCurrentTime)); + + len = sprintf(buff, "\n====time: %s DUMP START====\n", + szCurrentTime); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_global_vars(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_tracker_servers(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = fdfs_dump_groups_info(buff, sizeof(buff)); + WRITE_TO_FILE(fd, buff, len) + + len = sprintf(buff, "\ngroup name list:\n"); + WRITE_TO_FILE(fd, buff, len) + len = 0; + ppGroupEnd = g_groups.groups + g_groups.count; + for (ppGroup=g_groups.groups; ppGroupgroup_name); + } + len += sprintf(buff+len, "\n"); + WRITE_TO_FILE(fd, buff, len) + + ppGroupEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; ppGroup +#include +#include +#include "fdfs_define.h" +#include "tracker_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int fdfs_dump_tracker_global_vars_to_file(const char *filename); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_func.c b/tracker/tracker_func.c new file mode 100644 index 0000000..7693473 --- /dev/null +++ b/tracker/tracker_func.c @@ -0,0 +1,766 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_func.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "ini_file_reader.h" +#include "connection_pool.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_global.h" +#include "tracker_func.h" +#include "tracker_mem.h" +#include "local_ip_func.h" +#include "fdfs_shared_func.h" + +#ifdef WITH_HTTPD +#include "fdfs_http_shared.h" +#endif + +static int tracker_load_store_lookup(const char *filename, \ + IniContext *pItemContext) +{ + char *pGroupName; + g_groups.store_lookup = iniGetIntValue(NULL, "store_lookup", \ + pItemContext, FDFS_STORE_LOOKUP_ROUND_ROBIN); + if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN) + { + g_groups.store_group[0] = '\0'; + return 0; + } + + if (g_groups.store_lookup == FDFS_STORE_LOOKUP_LOAD_BALANCE) + { + g_groups.store_group[0] = '\0'; + return 0; + } + + if (g_groups.store_lookup != FDFS_STORE_LOOKUP_SPEC_GROUP) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", the value of \"store_lookup\" is " \ + "invalid, value=%d!", \ + __LINE__, filename, g_groups.store_lookup); + return EINVAL; + } + + pGroupName = iniGetStrValue(NULL, "store_group", pItemContext); + if (pGroupName == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" must have item " \ + "\"store_group\"!", __LINE__, filename); + return ENOENT; + } + if (pGroupName[0] == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "store_group is empty!", __LINE__, filename); + return EINVAL; + } + + snprintf(g_groups.store_group, sizeof(g_groups.store_group), \ + "%s", pGroupName); + if (fdfs_validate_group_name(g_groups.store_group) != 0) \ + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\", " \ + "the group name \"%s\" is invalid!", \ + __LINE__, filename, g_groups.store_group); + return EINVAL; + } + + return 0; +} + +static int tracker_load_storage_id_info(const char *config_filename, \ + IniContext *pItemContext) +{ + char *pIdType; + + g_use_storage_id = iniGetBoolValue(NULL, "use_storage_id", \ + pItemContext, false); + if (!g_use_storage_id) + { + return 0; + } + + pIdType = iniGetStrValue(NULL, "id_type_in_filename", \ + pItemContext); + if (pIdType != NULL && strcasecmp(pIdType, "id") == 0) + { + g_id_type_in_filename = FDFS_ID_TYPE_SERVER_ID; + } + else + { + g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS; + } + + return fdfs_load_storage_ids_from_file(config_filename, pItemContext); +} + +int tracker_load_from_conf_file(const char *filename, \ + char *bind_addr, const int addr_size) +{ + char *pBasePath; + char *pBindAddr; + char *pRunByGroup; + char *pRunByUser; + char *pThreadStackSize; + char *pSlotMinSize; + char *pSlotMaxSize; + char *pSpaceThreshold; + char *pTrunkFileSize; + char *pRotateErrorLogSize; +#ifdef WITH_HTTPD + char *pHttpCheckUri; + char *pHttpCheckType; +#endif + IniContext iniContext; + int result; + int64_t thread_stack_size; + int64_t trunk_file_size; + int64_t slot_min_size; + int64_t slot_max_size; + int64_t rotate_error_log_size; + char reserved_space_str[32]; + + memset(&g_groups, 0, sizeof(FDFSGroups)); + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load conf file \"%s\" fail, ret code: %d", \ + __LINE__, filename, result); + return result; + } + + //iniPrintItems(&iniContext); + + do + { + if (iniGetBoolValue(NULL, "disabled", &iniContext, false)) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" disabled=true, exit", \ + __LINE__, filename); + result = ECANCELED; + break; + } + + pBasePath = iniGetStrValue(NULL, "base_path", &iniContext); + if (pBasePath == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "conf file \"%s\" must have item " \ + "\"base_path\"!", __LINE__, filename); + result = ENOENT; + break; + } + + snprintf(g_fdfs_base_path, sizeof(g_fdfs_base_path), "%s", pBasePath); + chopPath(g_fdfs_base_path); + if (!fileExists(g_fdfs_base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" can't be accessed, error info: %s", \ + __LINE__, g_fdfs_base_path, STRERROR(errno)); + result = errno != 0 ? errno : ENOENT; + break; + } + if (!isDir(g_fdfs_base_path)) + { + logError("file: "__FILE__", line: %d, " \ + "\"%s\" is not a directory!", \ + __LINE__, g_fdfs_base_path); + result = ENOTDIR; + break; + } + + load_log_level(&iniContext); + if ((result=log_set_prefix(g_fdfs_base_path, \ + TRACKER_ERROR_LOG_FILENAME)) != 0) + { + break; + } + + g_fdfs_connect_timeout = iniGetIntValue(NULL, "connect_timeout", \ + &iniContext, DEFAULT_CONNECT_TIMEOUT); + if (g_fdfs_connect_timeout <= 0) + { + g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT; + } + + g_fdfs_network_timeout = iniGetIntValue(NULL, "network_timeout", \ + &iniContext, DEFAULT_NETWORK_TIMEOUT); + if (g_fdfs_network_timeout <= 0) + { + g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT; + } + g_network_tv.tv_sec = g_fdfs_network_timeout; + + g_server_port = iniGetIntValue(NULL, "port", &iniContext, \ + FDFS_TRACKER_SERVER_DEF_PORT); + if (g_server_port <= 0) + { + g_server_port = FDFS_TRACKER_SERVER_DEF_PORT; + } + + pBindAddr = iniGetStrValue(NULL, "bind_addr", &iniContext); + if (pBindAddr == NULL) + { + bind_addr[0] = '\0'; + } + else + { + snprintf(bind_addr, addr_size, "%s", pBindAddr); + } + + if ((result=tracker_load_store_lookup(filename, \ + &iniContext)) != 0) + { + break; + } + + g_groups.store_server = (byte)iniGetIntValue(NULL, \ + "store_server", &iniContext, \ + FDFS_STORE_SERVER_ROUND_ROBIN); + if (!(g_groups.store_server == FDFS_STORE_SERVER_FIRST_BY_IP ||\ + g_groups.store_server == FDFS_STORE_SERVER_FIRST_BY_PRI|| + g_groups.store_server == FDFS_STORE_SERVER_ROUND_ROBIN)) + { + logWarning("file: "__FILE__", line: %d, " \ + "store_server 's value %d is invalid, " \ + "set to %d (round robin)!", \ + __LINE__, g_groups.store_server, \ + FDFS_STORE_SERVER_ROUND_ROBIN); + + g_groups.store_server = FDFS_STORE_SERVER_ROUND_ROBIN; + } + + g_groups.download_server = (byte)iniGetIntValue(NULL, \ + "download_server", &iniContext, \ + FDFS_DOWNLOAD_SERVER_ROUND_ROBIN); + if (!(g_groups.download_server==FDFS_DOWNLOAD_SERVER_ROUND_ROBIN + || g_groups.download_server == + FDFS_DOWNLOAD_SERVER_SOURCE_FIRST)) + { + logWarning("file: "__FILE__", line: %d, " \ + "download_server 's value %d is invalid, " \ + "set to %d (round robin)!", \ + __LINE__, g_groups.download_server, \ + FDFS_DOWNLOAD_SERVER_ROUND_ROBIN); + + g_groups.download_server = \ + FDFS_DOWNLOAD_SERVER_ROUND_ROBIN; + } + + g_groups.store_path = (byte)iniGetIntValue(NULL, "store_path", \ + &iniContext, FDFS_STORE_PATH_ROUND_ROBIN); + if (!(g_groups.store_path == FDFS_STORE_PATH_ROUND_ROBIN || \ + g_groups.store_path == FDFS_STORE_PATH_LOAD_BALANCE)) + { + logWarning("file: "__FILE__", line: %d, " \ + "store_path 's value %d is invalid, " \ + "set to %d (round robin)!", \ + __LINE__, g_groups.store_path , \ + FDFS_STORE_PATH_ROUND_ROBIN); + g_groups.store_path = FDFS_STORE_PATH_ROUND_ROBIN; + } + + if ((result=fdfs_parse_storage_reserved_space(&iniContext, \ + &g_storage_reserved_space)) != 0) + { + break; + } + + g_max_connections = iniGetIntValue(NULL, "max_connections", \ + &iniContext, DEFAULT_MAX_CONNECTONS); + if (g_max_connections <= 0) + { + g_max_connections = DEFAULT_MAX_CONNECTONS; + } + + g_accept_threads = iniGetIntValue(NULL, "accept_threads", \ + &iniContext, 1); + if (g_accept_threads <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"accept_threads\" is invalid, " \ + "value: %d <= 0!", __LINE__, g_accept_threads); + result = EINVAL; + break; + } + + g_work_threads = iniGetIntValue(NULL, "work_threads", \ + &iniContext, DEFAULT_WORK_THREADS); + if (g_work_threads <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"work_threads\" is invalid, " \ + "value: %d <= 0!", __LINE__, g_work_threads); + result = EINVAL; + break; + } + + if ((result=set_rlimit(RLIMIT_NOFILE, g_max_connections)) != 0) + { + break; + } + + pRunByGroup = iniGetStrValue(NULL, "run_by_group", &iniContext); + pRunByUser = iniGetStrValue(NULL, "run_by_user", &iniContext); + if (pRunByGroup == NULL) + { + *g_run_by_group = '\0'; + } + else + { + snprintf(g_run_by_group, sizeof(g_run_by_group), \ + "%s", pRunByGroup); + } + if (*g_run_by_group == '\0') + { + g_run_by_gid = getegid(); + } + else + { + struct group *pGroup; + + pGroup = getgrnam(g_run_by_group); + if (pGroup == NULL) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "getgrnam fail, errno: %d, " \ + "error info: %s", __LINE__, \ + result, STRERROR(result)); + return result; + } + + g_run_by_gid = pGroup->gr_gid; + } + + + if (pRunByUser == NULL) + { + *g_run_by_user = '\0'; + } + else + { + snprintf(g_run_by_user, sizeof(g_run_by_user), \ + "%s", pRunByUser); + } + if (*g_run_by_user == '\0') + { + g_run_by_uid = geteuid(); + } + else + { + struct passwd *pUser; + + pUser = getpwnam(g_run_by_user); + if (pUser == NULL) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "getpwnam fail, errno: %d, " \ + "error info: %s", __LINE__, \ + result, STRERROR(result)); + return result; + } + + g_run_by_uid = pUser->pw_uid; + } + + if ((result=load_allow_hosts(&iniContext, \ + &g_allow_ip_addrs, &g_allow_ip_count)) != 0) + { + return result; + } + + g_sync_log_buff_interval = iniGetIntValue(NULL, \ + "sync_log_buff_interval", &iniContext, \ + SYNC_LOG_BUFF_DEF_INTERVAL); + if (g_sync_log_buff_interval <= 0) + { + g_sync_log_buff_interval = SYNC_LOG_BUFF_DEF_INTERVAL; + } + + g_check_active_interval = iniGetIntValue(NULL, \ + "check_active_interval", &iniContext, \ + CHECK_ACTIVE_DEF_INTERVAL); + if (g_check_active_interval <= 0) + { + g_check_active_interval = CHECK_ACTIVE_DEF_INTERVAL; + } + + pThreadStackSize = iniGetStrValue(NULL, \ + "thread_stack_size", &iniContext); + if (pThreadStackSize == NULL) + { + thread_stack_size = 64 * 1024; + } + else if ((result=parse_bytes(pThreadStackSize, 1, \ + &thread_stack_size)) != 0) + { + return result; + } + g_thread_stack_size = (int)thread_stack_size; + + g_storage_ip_changed_auto_adjust = iniGetBoolValue(NULL, \ + "storage_ip_changed_auto_adjust", \ + &iniContext, true); + + g_storage_sync_file_max_delay = iniGetIntValue(NULL, \ + "storage_sync_file_max_delay", &iniContext, \ + DEFAULT_STORAGE_SYNC_FILE_MAX_DELAY); + if (g_storage_sync_file_max_delay <= 0) + { + g_storage_sync_file_max_delay = \ + DEFAULT_STORAGE_SYNC_FILE_MAX_DELAY; + } + + g_storage_sync_file_max_time = iniGetIntValue(NULL, \ + "storage_sync_file_max_time", &iniContext, \ + DEFAULT_STORAGE_SYNC_FILE_MAX_TIME); + if (g_storage_sync_file_max_time <= 0) + { + g_storage_sync_file_max_time = \ + DEFAULT_STORAGE_SYNC_FILE_MAX_TIME; + } + + g_if_use_trunk_file = iniGetBoolValue(NULL, \ + "use_trunk_file", &iniContext, false); + + pSlotMinSize = iniGetStrValue(NULL, \ + "slot_min_size", &iniContext); + if (pSlotMinSize == NULL) + { + slot_min_size = 256; + } + else if ((result=parse_bytes(pSlotMinSize, 1, \ + &slot_min_size)) != 0) + { + return result; + } + g_slot_min_size = (int)slot_min_size; + if (g_slot_min_size <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "item \"slot_min_size\" %d is invalid, " \ + "which <= 0", __LINE__, g_slot_min_size); + result = EINVAL; + break; + } + if (g_slot_min_size > 64 * 1024) + { + logWarning("file: "__FILE__", line: %d, " \ + "item \"slot_min_size\" %d is too large, " \ + "change to 64KB", __LINE__, g_slot_min_size); + g_slot_min_size = 64 * 1024; + } + + pTrunkFileSize = iniGetStrValue(NULL, \ + "trunk_file_size", &iniContext); + if (pTrunkFileSize == NULL) + { + trunk_file_size = 64 * 1024 * 1024; + } + else if ((result=parse_bytes(pTrunkFileSize, 1, \ + &trunk_file_size)) != 0) + { + return result; + } + g_trunk_file_size = (int)trunk_file_size; + if (g_trunk_file_size < 4 * 1024 * 1024) + { + logWarning("file: "__FILE__", line: %d, " \ + "item \"trunk_file_size\" %d is too small, " \ + "change to 4MB", __LINE__, g_trunk_file_size); + g_trunk_file_size = 4 * 1024 * 1024; + } + + pSlotMaxSize = iniGetStrValue(NULL, \ + "slot_max_size", &iniContext); + if (pSlotMaxSize == NULL) + { + slot_max_size = g_trunk_file_size / 2; + } + else if ((result=parse_bytes(pSlotMaxSize, 1, \ + &slot_max_size)) != 0) + { + return result; + } + g_slot_max_size = (int)slot_max_size; + if (g_slot_max_size <= g_slot_min_size) + { + logError("file: "__FILE__", line: %d, " \ + "item \"slot_max_size\" %d is invalid, " \ + "which <= slot_min_size: %d", \ + __LINE__, g_slot_max_size, g_slot_min_size); + result = EINVAL; + break; + } + if (g_slot_max_size > g_trunk_file_size / 2) + { + logWarning("file: "__FILE__", line: %d, " \ + "item \"slot_max_size\": %d is too large, " \ + "change to %d", __LINE__, g_slot_max_size, \ + g_trunk_file_size / 2); + g_slot_max_size = g_trunk_file_size / 2; + } + + g_trunk_create_file_advance = iniGetBoolValue(NULL, \ + "trunk_create_file_advance", &iniContext, false); + if ((result=get_time_item_from_conf(&iniContext, \ + "trunk_create_file_time_base", \ + &g_trunk_create_file_time_base, 2, 0)) != 0) + { + return result; + } + + g_trunk_create_file_interval = iniGetIntValue(NULL, \ + "trunk_create_file_interval", &iniContext, \ + 86400); + pSpaceThreshold = iniGetStrValue(NULL, \ + "trunk_create_file_space_threshold", &iniContext); + if (pSpaceThreshold == NULL) + { + g_trunk_create_file_space_threshold = 0; + } + else if ((result=parse_bytes(pSpaceThreshold, 1, \ + &g_trunk_create_file_space_threshold)) != 0) + { + return result; + } + g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL, \ + "trunk_compress_binlog_min_interval", \ + &iniContext, 0); + + g_trunk_init_check_occupying = iniGetBoolValue(NULL, \ + "trunk_init_check_occupying", &iniContext, false); + + g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL, \ + "trunk_init_reload_from_binlog", &iniContext, false); + + if ((result=tracker_load_storage_id_info( \ + filename, &iniContext)) != 0) + { + return result; + } + + g_rotate_error_log = iniGetBoolValue(NULL, "rotate_error_log",\ + &iniContext, false); + if ((result=get_time_item_from_conf(&iniContext, \ + "error_log_rotate_time", &g_error_log_rotate_time, \ + 0, 0)) != 0) + { + break; + } + + pRotateErrorLogSize = iniGetStrValue(NULL, \ + "rotate_error_log_size", &iniContext); + if (pRotateErrorLogSize == NULL) + { + rotate_error_log_size = 0; + } + else if ((result=parse_bytes(pRotateErrorLogSize, 1, \ + &rotate_error_log_size)) != 0) + { + break; + } + if (rotate_error_log_size > 0 && \ + rotate_error_log_size < FDFS_ONE_MB) + { + logWarning("file: "__FILE__", line: %d, " \ + "item \"rotate_error_log_size\": " \ + INT64_PRINTF_FORMAT" is too small, " \ + "change to 1 MB", __LINE__, \ + rotate_error_log_size); + rotate_error_log_size = FDFS_ONE_MB; + } + g_log_context.rotate_size = rotate_error_log_size; + + g_store_slave_file_use_link = iniGetBoolValue(NULL, \ + "store_slave_file_use_link", &iniContext, false); + + if ((result=fdfs_connection_pool_init(filename, &iniContext)) != 0) + { + break; + } + +#ifdef WITH_HTTPD + if ((result=fdfs_http_params_load(&iniContext, \ + filename, &g_http_params)) != 0) + { + return result; + } + + g_http_check_interval = iniGetIntValue(NULL, \ + "http.check_alive_interval", &iniContext, 30); + + pHttpCheckType = iniGetStrValue(NULL, \ + "http.check_alive_type", &iniContext); + if (pHttpCheckType != NULL && \ + strcasecmp(pHttpCheckType, "http") == 0) + { + g_http_check_type = FDFS_HTTP_CHECK_ALIVE_TYPE_HTTP; + } + else + { + g_http_check_type = FDFS_HTTP_CHECK_ALIVE_TYPE_TCP; + } + + pHttpCheckUri = iniGetStrValue(NULL, \ + "http.check_alive_uri", &iniContext); + if (pHttpCheckUri == NULL) + { + *g_http_check_uri = '/'; + *(g_http_check_uri+1) = '\0'; + } + else if (*pHttpCheckUri == '/') + { + snprintf(g_http_check_uri, sizeof(g_http_check_uri), \ + "%s", pHttpCheckUri); + } + else + { + snprintf(g_http_check_uri, sizeof(g_http_check_uri), \ + "/%s", pHttpCheckUri); + } + + +#endif + + logInfo("FastDFS v%d.%02d, base_path=%s, " \ + "run_by_group=%s, run_by_user=%s, " \ + "connect_timeout=%ds, " \ + "network_timeout=%ds, " \ + "port=%d, bind_addr=%s, " \ + "max_connections=%d, " \ + "accept_threads=%d, " \ + "work_threads=%d, " \ + "store_lookup=%d, store_group=%s, " \ + "store_server=%d, store_path=%d, " \ + "reserved_storage_space=%s, " \ + "download_server=%d, " \ + "allow_ip_count=%d, sync_log_buff_interval=%ds, " \ + "check_active_interval=%ds, " \ + "thread_stack_size=%d KB, " \ + "storage_ip_changed_auto_adjust=%d, " \ + "storage_sync_file_max_delay=%ds, " \ + "storage_sync_file_max_time=%ds, " \ + "use_trunk_file=%d, " \ + "slot_min_size=%d, " \ + "slot_max_size=%d MB, " \ + "trunk_file_size=%d MB, " \ + "trunk_create_file_advance=%d, " \ + "trunk_create_file_time_base=%02d:%02d, " \ + "trunk_create_file_interval=%d, " \ + "trunk_create_file_space_threshold=%d GB, " \ + "trunk_init_check_occupying=%d, " \ + "trunk_init_reload_from_binlog=%d, " \ + "trunk_compress_binlog_min_interval=%d, " \ + "use_storage_id=%d, " \ + "id_type_in_filename=%s, " \ + "storage_id_count=%d, " \ + "rotate_error_log=%d, " \ + "error_log_rotate_time=%02d:%02d, " \ + "rotate_error_log_size="INT64_PRINTF_FORMAT", " \ + "store_slave_file_use_link=%d, " \ + "use_connection_pool=%d, " \ + "g_connection_pool_max_idle_time=%ds", \ + g_fdfs_version.major, g_fdfs_version.minor, \ + g_fdfs_base_path, g_run_by_group, g_run_by_user, \ + g_fdfs_connect_timeout, \ + g_fdfs_network_timeout, g_server_port, bind_addr, \ + g_max_connections, g_accept_threads, g_work_threads, \ + g_groups.store_lookup, g_groups.store_group, \ + g_groups.store_server, g_groups.store_path, \ + fdfs_storage_reserved_space_to_string( \ + &g_storage_reserved_space, reserved_space_str), \ + g_groups.download_server, \ + g_allow_ip_count, g_sync_log_buff_interval, \ + g_check_active_interval, g_thread_stack_size / 1024, \ + g_storage_ip_changed_auto_adjust, \ + g_storage_sync_file_max_delay, \ + g_storage_sync_file_max_time, \ + g_if_use_trunk_file, g_slot_min_size, \ + g_slot_max_size / FDFS_ONE_MB, \ + g_trunk_file_size / FDFS_ONE_MB, \ + g_trunk_create_file_advance, \ + g_trunk_create_file_time_base.hour, \ + g_trunk_create_file_time_base.minute, \ + g_trunk_create_file_interval, \ + (int)(g_trunk_create_file_space_threshold / \ + (FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying, \ + g_trunk_init_reload_from_binlog, \ + g_trunk_compress_binlog_min_interval, \ + g_use_storage_id, g_id_type_in_filename == \ + FDFS_ID_TYPE_SERVER_ID ? "id" : "ip", g_storage_id_count, \ + g_rotate_error_log, g_error_log_rotate_time.hour, \ + g_error_log_rotate_time.minute, \ + g_log_context.rotate_size, g_store_slave_file_use_link, \ + g_use_connection_pool, g_connection_pool_max_idle_time); + +#ifdef WITH_HTTPD + if (!g_http_params.disabled) + { + logInfo("HTTP supported: " \ + "server_port=%d, " \ + "default_content_type=%s, " \ + "anti_steal_token=%d, " \ + "token_ttl=%ds, " \ + "anti_steal_secret_key length=%d, " \ + "token_check_fail content_type=%s, " \ + "token_check_fail buff length=%d, " \ + "check_active_interval=%d, " \ + "check_active_type=%s, " \ + "check_active_uri=%s", \ + g_http_params.server_port, \ + g_http_params.default_content_type, \ + g_http_params.anti_steal_token, \ + g_http_params.token_ttl, \ + g_http_params.anti_steal_secret_key.length, \ + g_http_params.token_check_fail_content_type, \ + g_http_params.token_check_fail_buff.length, \ + g_http_check_interval, g_http_check_type == \ + FDFS_HTTP_CHECK_ALIVE_TYPE_TCP ? "tcp":"http",\ + g_http_check_uri); + } +#endif + + } while (0); + + iniFreeContext(&iniContext); + + load_local_host_ip_addrs(); + + return result; +} + diff --git a/tracker/tracker_func.h b/tracker/tracker_func.h new file mode 100644 index 0000000..cb044f6 --- /dev/null +++ b/tracker/tracker_func.h @@ -0,0 +1,27 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_func.h + +#ifndef _TRACKER_FUNC_H_ +#define _TRACKER_FUNC_H_ + +#include "tracker_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int tracker_load_from_conf_file(const char *filename, \ + char *bind_addr, const int addr_size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_global.c b/tracker/tracker_global.c new file mode 100644 index 0000000..4caaa09 --- /dev/null +++ b/tracker/tracker_global.c @@ -0,0 +1,75 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include "tracker_global.h" + +volatile bool g_continue_flag = true; +int g_server_port = FDFS_TRACKER_SERVER_DEF_PORT; +int g_max_connections = DEFAULT_MAX_CONNECTONS; +int g_accept_threads = 1; +int g_work_threads = DEFAULT_WORK_THREADS; +int g_sync_log_buff_interval = SYNC_LOG_BUFF_DEF_INTERVAL; +int g_check_active_interval = CHECK_ACTIVE_DEF_INTERVAL; + +struct timeval g_network_tv = {DEFAULT_NETWORK_TIMEOUT, 0}; + +FDFSGroups g_groups; +int g_storage_stat_chg_count = 0; +int g_storage_sync_time_chg_count = 0; //sync timestamp +FDFSStorageReservedSpace g_storage_reserved_space = { \ + TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB}; + +int g_allow_ip_count = 0; +in_addr_t *g_allow_ip_addrs = NULL; + +struct base64_context g_base64_context; + +gid_t g_run_by_gid; +uid_t g_run_by_uid; + +char g_run_by_group[32] = {0}; +char g_run_by_user[32] = {0}; + +bool g_storage_ip_changed_auto_adjust = true; +bool g_use_storage_id = false; //if use storage ID instead of IP address +byte g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS; //id type of the storage server in the filename +bool g_rotate_error_log = false; //if rotate the error log every day +TimeInfo g_error_log_rotate_time = {0, 0}; //rotate error log time base + +int g_thread_stack_size = 64 * 1024; +int g_storage_sync_file_max_delay = DEFAULT_STORAGE_SYNC_FILE_MAX_DELAY; +int g_storage_sync_file_max_time = DEFAULT_STORAGE_SYNC_FILE_MAX_TIME; + +bool g_store_slave_file_use_link = false; //if store slave file use symbol link +bool g_if_use_trunk_file = false; //if use trunk file +bool g_trunk_create_file_advance = false; +bool g_trunk_init_check_occupying = false; +bool g_trunk_init_reload_from_binlog = false; +int g_slot_min_size = 256; //slot min size, such as 256 bytes +int g_slot_max_size = 16 * 1024 * 1024; //slot max size, such as 16MB +int g_trunk_file_size = 64 * 1024 * 1024; //the trunk file size, such as 64MB +TimeInfo g_trunk_create_file_time_base = {0, 0}; +int g_trunk_create_file_interval = 86400; +int g_trunk_compress_binlog_min_interval = 0; +int64_t g_trunk_create_file_space_threshold = 0; + +time_t g_up_time = 0; +TrackerStatus g_tracker_last_status = {0, 0}; + +#ifdef WITH_HTTPD +FDFSHTTPParams g_http_params; +int g_http_check_interval = 30; +int g_http_check_type = FDFS_HTTP_CHECK_ALIVE_TYPE_TCP; +char g_http_check_uri[128] = {0}; +bool g_http_servers_dirty = false; +#endif + +#if defined(DEBUG_FLAG) && defined(OS_LINUX) +char g_exe_name[256] = {0}; +#endif + diff --git a/tracker/tracker_global.h b/tracker/tracker_global.h new file mode 100644 index 0000000..69b4d67 --- /dev/null +++ b/tracker/tracker_global.h @@ -0,0 +1,104 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_global.h + +#ifndef _TRACKER_GLOBAL_H +#define _TRACKER_GLOBAL_H + +#include +#include +#include +#include +#include "common_define.h" +#include "fdfs_define.h" +#include "tracker_types.h" +#include "tracker_status.h" +#include "base64.h" +#include "hash.h" +#include "local_ip_func.h" + +#ifdef WITH_HTTPD +#include "fdfs_http_shared.h" +#endif + +#define TRACKER_SYNC_TO_FILE_FREQ 1000 +#define TRACKER_MAX_PACKAGE_SIZE (8 * 1024) +#define TRACKER_SYNC_STATUS_FILE_INTERVAL 3600 //one hour + +#ifdef __cplusplus +extern "C" { +#endif + +extern volatile bool g_continue_flag; +extern int g_server_port; +extern FDFSGroups g_groups; +extern int g_storage_stat_chg_count; +extern int g_storage_sync_time_chg_count; //sync timestamp +extern int g_max_connections; +extern int g_accept_threads; +extern int g_work_threads; +extern FDFSStorageReservedSpace g_storage_reserved_space; +extern int g_sync_log_buff_interval; //sync log buff to disk every interval seconds +extern int g_check_active_interval; //check storage server alive every interval seconds + +extern struct timeval g_network_tv; + +extern int g_allow_ip_count; /* -1 means match any ip address */ +extern in_addr_t *g_allow_ip_addrs; /* sorted array, asc order */ +extern struct base64_context g_base64_context; + +extern gid_t g_run_by_gid; +extern uid_t g_run_by_uid; + +extern char g_run_by_group[32]; +extern char g_run_by_user[32]; + +extern bool g_storage_ip_changed_auto_adjust; +extern bool g_use_storage_id; //identify storage by ID instead of IP address +extern byte g_id_type_in_filename; //id type of the storage server in the filename +extern bool g_rotate_error_log; //if rotate the error log every day +extern TimeInfo g_error_log_rotate_time; //rotate error log time base + +extern int g_thread_stack_size; +extern int g_storage_sync_file_max_delay; +extern int g_storage_sync_file_max_time; + +extern bool g_store_slave_file_use_link; //if store slave file use symbol link +extern bool g_if_use_trunk_file; //if use trunk file +extern bool g_trunk_create_file_advance; +extern bool g_trunk_init_check_occupying; +extern bool g_trunk_init_reload_from_binlog; +extern int g_slot_min_size; //slot min size, such as 256 bytes +extern int g_slot_max_size; //slot max size, such as 16MB +extern int g_trunk_file_size; //the trunk file size, such as 64MB +extern TimeInfo g_trunk_create_file_time_base; +extern int g_trunk_create_file_interval; +extern int g_trunk_compress_binlog_min_interval; +extern int64_t g_trunk_create_file_space_threshold; + +extern time_t g_up_time; +extern TrackerStatus g_tracker_last_status; //the status of last running + +#ifdef WITH_HTTPD +extern FDFSHTTPParams g_http_params; +extern int g_http_check_interval; +extern int g_http_check_type; +extern char g_http_check_uri[128]; +extern bool g_http_servers_dirty; +#endif + +#if defined(DEBUG_FLAG) && defined(OS_LINUX) +extern char g_exe_name[256]; +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_http_check.c b/tracker/tracker_http_check.c new file mode 100644 index 0000000..db0032d --- /dev/null +++ b/tracker/tracker_http_check.c @@ -0,0 +1,321 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.h" +#include "fdfs_global.h" +#include "tracker_global.h" +#include "tracker_mem.h" +#include "tracker_proto.h" +#include "http_func.h" +#include "sockopt.h" +#include "tracker_http_check.h" + +static pthread_t http_check_tid; +bool g_http_check_flag = false; + +static void *http_check_entrance(void *arg) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + char url[512]; + char error_info[512]; + char *content; + int content_len; + int http_status; + int sock; + int server_count; + int result; + + g_http_check_flag = true; + g_http_servers_dirty = false; + while (g_continue_flag) + { + if (g_http_servers_dirty) + { + g_http_servers_dirty = false; + } + else + { + sleep(g_http_check_interval); + } + + ppGroupEnd = g_groups.groups + g_groups.count; + for (ppGroup=g_groups.groups; g_continue_flag && (!g_http_servers_dirty)\ + && ppGroupstorage_http_port <= 0) + { + continue; + } + + server_count = 0; + ppServerEnd = (*ppGroup)->active_servers + (*ppGroup)->active_count; + for (ppServer=(*ppGroup)->active_servers; g_continue_flag && \ + (!g_http_servers_dirty) && ppServerip_addr, \ + (*ppGroup)->storage_http_port, \ + g_fdfs_connect_timeout); + close(sock); + + if (g_http_servers_dirty) + { + break; + } + + if (result == 0) + { + *((*ppGroup)->http_servers+server_count)=*ppServer; + server_count++; + if ((*ppServer)->http_check_fail_count > 0) + { + logInfo("file: "__FILE__", line: %d, " \ + "http check alive success " \ + "after %d times, server: %s:%d", + __LINE__, \ + (*ppServer)->http_check_fail_count, + (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port); + (*ppServer)->http_check_fail_count = 0; + } + } + else + { + if (result != (*ppServer)->http_check_last_errno) + { + if ((*ppServer)->http_check_fail_count > 1) + { + logError("file: "__FILE__", line: %d, "\ + "http check alive fail after " \ + "%d times, storage server: " \ + "%s:%d, error info: %s", \ + __LINE__, \ + (*ppServer)->http_check_fail_count, \ + (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port, \ + (*ppServer)->http_check_error_info); + } + + sprintf((*ppServer)->http_check_error_info, + "http check alive, connect to http " \ + "server %s:%d fail, " \ + "errno: %d, error info: %s", \ + (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port, result, \ + STRERROR(result)); + + logError("file: "__FILE__", line: %d, %s", \ + __LINE__, \ + (*ppServer)->http_check_error_info); + (*ppServer)->http_check_last_errno = result; + (*ppServer)->http_check_fail_count = 1; + } + else + { + (*ppServer)->http_check_fail_count++; + } + } + } + else //http + { + sprintf(url, "http://%s:%d%s", (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port, g_http_check_uri); + + result = get_url_content(url, g_fdfs_connect_timeout, \ + g_fdfs_network_timeout, &http_status, \ + &content, &content_len, error_info); + + if (g_http_servers_dirty) + { + if (result == 0) + { + free(content); + } + + break; + } + + if (result == 0) + { + if (http_status == 200) + { + *((*ppGroup)->http_servers+server_count)=*ppServer; + server_count++; + + if ((*ppServer)->http_check_fail_count > 0) + { + logInfo("file: "__FILE__", line: %d, " \ + "http check alive success " \ + "after %d times, url: %s",\ + __LINE__, \ + (*ppServer)->http_check_fail_count, + url); + (*ppServer)->http_check_fail_count = 0; + } + } + else + { + if (http_status != (*ppServer)->http_check_last_status) + { + if ((*ppServer)->http_check_fail_count > 1) + { + logError("file: "__FILE__", line: %d, "\ + "http check alive fail after " \ + "%d times, storage server: " \ + "%s:%d, error info: %s", \ + __LINE__, \ + (*ppServer)->http_check_fail_count, \ + (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port, \ + (*ppServer)->http_check_error_info); + } + + sprintf((*ppServer)->http_check_error_info, \ + "http check alive fail, url: %s, " \ + "http_status=%d", url, http_status); + + logError("file: "__FILE__", line: %d, %s", \ + __LINE__, \ + (*ppServer)->http_check_error_info); + (*ppServer)->http_check_last_status = http_status; + (*ppServer)->http_check_fail_count = 1; + } + else + { + (*ppServer)->http_check_fail_count++; + } + } + + free(content); + } + else + { + if (result != (*ppServer)->http_check_last_errno) + { + if ((*ppServer)->http_check_fail_count > 1) + { + logError("file: "__FILE__", line: %d, "\ + "http check alive fail after " \ + "%d times, storage server: " \ + "%s:%d, error info: %s", \ + __LINE__, \ + (*ppServer)->http_check_fail_count, \ + (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port, \ + (*ppServer)->http_check_error_info); + } + + sprintf((*ppServer)->http_check_error_info, \ + "http check alive fail, " \ + "error info: %s", error_info); + + logError("file: "__FILE__", line: %d, %s", \ + __LINE__, \ + (*ppServer)->http_check_error_info); + (*ppServer)->http_check_last_errno = result; + (*ppServer)->http_check_fail_count = 1; + } + else + { + (*ppServer)->http_check_fail_count++; + } + } + } + } + + if (g_http_servers_dirty) + { + break; + } + + if ((*ppGroup)->http_server_count != server_count) + { + logDebug("file: "__FILE__", line: %d, " \ + "group: %s, HTTP server count change from %d to %d", \ + __LINE__, (*ppGroup)->group_name, \ + (*ppGroup)->http_server_count, server_count); + + (*ppGroup)->http_server_count = server_count; + } + } + } + + ppGroupEnd = g_groups.groups + g_groups.count; + for (ppGroup=g_groups.groups; ppGroupall_servers + (*ppGroup)->count; + for (ppServer=(*ppGroup)->all_servers; ppServerhttp_check_fail_count > 1) + { + logError("file: "__FILE__", line: %d, " \ + "http check alive fail " \ + "after %d times, storage server: %s:%d, " \ + "error info: %s", \ + __LINE__, (*ppServer)->http_check_fail_count, \ + (*ppServer)->ip_addr, \ + (*ppGroup)->storage_http_port,\ + (*ppServer)->http_check_error_info); + } + } + } + + g_http_check_flag = false; + return NULL; +} + +int tracker_http_check_start() +{ + int result; + + if (g_http_check_interval <= 0) + { + return 0; + } + + if ((result=pthread_create(&http_check_tid, NULL, \ + http_check_entrance, NULL)) != 0) + { + logCrit("file: "__FILE__", line: %d, " \ + "create thread failed, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int tracker_http_check_stop() +{ + if (g_http_check_interval <= 0) + { + return 0; + } + + return pthread_kill(http_check_tid, SIGINT); +} + diff --git a/tracker/tracker_http_check.h b/tracker/tracker_http_check.h new file mode 100644 index 0000000..6bfd23a --- /dev/null +++ b/tracker/tracker_http_check.h @@ -0,0 +1,27 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_httpd.h + +#ifndef _TRACKER_HTTP_CHECK_H_ +#define _TRACKER_HTTP_CHECK_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern bool g_http_check_flag; + +int tracker_http_check_start(); +int tracker_http_check_stop(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_mem.c b/tracker/tracker_mem.c new file mode 100644 index 0000000..3c6d151 --- /dev/null +++ b/tracker/tracker_mem.c @@ -0,0 +1,6006 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "sockopt.h" +#include "fdfs_global.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "fdfs_shared_func.h" +#include "tracker_global.h" +#include "tracker_proto.h" +#include "tracker_func.h" +#include "tracker_relationship.h" +#include "tracker_mem.h" + +#define TRACKER_MEM_ALLOC_ONCE 2 + +#define GROUP_SECTION_NAME_GLOBAL "Global" +#define GROUP_SECTION_NAME_PREFIX "Group" +#define GROUP_SECTION_NO_FORMAT "%03d" +#define GROUP_ITEM_GROUP_COUNT "group_count" +#define GROUP_ITEM_GROUP_NAME "group_name" +#define GROUP_ITEM_STORAGE_PORT "storage_port" +#define GROUP_ITEM_STORAGE_HTTP_PORT "storage_http_port" +#define GROUP_ITEM_STORE_PATH_COUNT "store_path_count" +#define GROUP_ITEM_SUBDIR_COUNT_PER_PATH "subdir_count_per_path" +#define GROUP_ITEM_CURRENT_TRUNK_FILE_ID "current_trunk_file_id" +#define GROUP_ITEM_LAST_TRUNK_SERVER "last_trunk_server" +#define GROUP_ITEM_TRUNK_SERVER "trunk_server" + +#define STORAGE_SECTION_NAME_GLOBAL "Global" +#define STORAGE_SECTION_NAME_PREFIX "Storage" +#define STORAGE_SECTION_NO_FORMAT "%03d" +#define STORAGE_ITEM_STORAGE_COUNT "storage_count" + +#define STORAGE_ITEM_GROUP_NAME "group_name" +#define STORAGE_ITEM_SERVER_ID "id" +#define STORAGE_ITEM_IP_ADDR "ip_addr" +#define STORAGE_ITEM_STATUS "status" +#define STORAGE_ITEM_DOMAIN_NAME "domain_name" +#define STORAGE_ITEM_VERSION "version" +#define STORAGE_ITEM_SYNC_SRC_SERVER "sync_src_server" +#define STORAGE_ITEM_SYNC_UNTIL_TIMESTAMP "sync_until_timestamp" +#define STORAGE_ITEM_JOIN_TIME "join_time" +#define STORAGE_ITEM_TOTAL_MB "total_mb" +#define STORAGE_ITEM_FREE_MB "free_mb" +#define STORAGE_ITEM_CHANGELOG_OFFSET "changelog_offset" +#define STORAGE_ITEM_STORE_PATH_COUNT "store_path_count" +#define STORAGE_ITEM_SUBDIR_COUNT_PER_PATH "subdir_count_per_path" +#define STORAGE_ITEM_UPLOAD_PRIORITY "upload_priority" +#define STORAGE_ITEM_STORAGE_PORT "storage_port" +#define STORAGE_ITEM_STORAGE_HTTP_PORT "storage_http_port" +#define STORAGE_ITEM_TOTAL_UPLOAD_COUNT "total_upload_count" +#define STORAGE_ITEM_SUCCESS_UPLOAD_COUNT "success_upload_count" +#define STORAGE_ITEM_TOTAL_APPEND_COUNT "total_append_count" +#define STORAGE_ITEM_SUCCESS_APPEND_COUNT "success_append_count" +#define STORAGE_ITEM_TOTAL_SET_META_COUNT "total_set_meta_count" +#define STORAGE_ITEM_SUCCESS_SET_META_COUNT "success_set_meta_count" +#define STORAGE_ITEM_TOTAL_DELETE_COUNT "total_delete_count" +#define STORAGE_ITEM_SUCCESS_DELETE_COUNT "success_delete_count" +#define STORAGE_ITEM_TOTAL_DOWNLOAD_COUNT "total_download_count" +#define STORAGE_ITEM_SUCCESS_DOWNLOAD_COUNT "success_download_count" +#define STORAGE_ITEM_TOTAL_GET_META_COUNT "total_get_meta_count" +#define STORAGE_ITEM_SUCCESS_GET_META_COUNT "success_get_meta_count" +#define STORAGE_ITEM_TOTAL_CREATE_LINK_COUNT "total_create_link_count" +#define STORAGE_ITEM_SUCCESS_CREATE_LINK_COUNT "success_create_link_count" +#define STORAGE_ITEM_TOTAL_DELETE_LINK_COUNT "total_delete_link_count" +#define STORAGE_ITEM_SUCCESS_DELETE_LINK_COUNT "success_delete_link_count" +#define STORAGE_ITEM_TOTAL_UPLOAD_BYTES "total_upload_bytes" +#define STORAGE_ITEM_SUCCESS_UPLOAD_BYTES "success_upload_bytes" +#define STORAGE_ITEM_TOTAL_APPEND_BYTES "total_append_bytes" +#define STORAGE_ITEM_SUCCESS_APPEND_BYTES "success_append_bytes" +#define STORAGE_ITEM_TOTAL_DOWNLOAD_BYTES "total_download_bytes" +#define STORAGE_ITEM_SUCCESS_DOWNLOAD_BYTES "success_download_bytes" +#define STORAGE_ITEM_TOTAL_SYNC_IN_BYTES "total_sync_in_bytes" +#define STORAGE_ITEM_SUCCESS_SYNC_IN_BYTES "success_sync_in_bytes" +#define STORAGE_ITEM_TOTAL_SYNC_OUT_BYTES "total_sync_out_bytes" +#define STORAGE_ITEM_SUCCESS_SYNC_OUT_BYTES "success_sync_out_bytes" +#define STORAGE_ITEM_TOTAL_FILE_OPEN_COUNT "total_file_open_count" +#define STORAGE_ITEM_SUCCESS_FILE_OPEN_COUNT "success_file_open_count" +#define STORAGE_ITEM_TOTAL_FILE_READ_COUNT "total_file_read_count" +#define STORAGE_ITEM_SUCCESS_FILE_READ_COUNT "success_file_read_count" +#define STORAGE_ITEM_TOTAL_FILE_WRITE_COUNT "total_file_write_count" +#define STORAGE_ITEM_SUCCESS_FILE_WRITE_COUNT "success_file_write_count" +#define STORAGE_ITEM_LAST_SOURCE_UPDATE "last_source_update" +#define STORAGE_ITEM_LAST_SYNC_UPDATE "last_sync_update" +#define STORAGE_ITEM_LAST_SYNCED_TIMESTAMP "last_synced_timestamp" +#define STORAGE_ITEM_LAST_HEART_BEAT_TIME "last_heart_beat_time" + +TrackerServerGroup g_tracker_servers = {0, 0, -1, NULL}; +ConnectionInfo *g_last_tracker_servers = NULL; //for delay free +int g_next_leader_index = -1; //next leader index +int g_trunk_server_chg_count = 1; //for notify other trackers +int g_tracker_leader_chg_count = 0; //for notify storage servers + +int64_t g_changelog_fsize = 0; //storage server change log file size +static int changelog_fd = -1; //storage server change log fd for write +static bool need_get_sys_files = true; +static bool get_sys_files_done = false; + +static pthread_mutex_t mem_thread_lock; +static pthread_mutex_t mem_file_lock; + +static void tracker_mem_find_store_server(FDFSGroupInfo *pGroup); +static int tracker_mem_find_trunk_server(FDFSGroupInfo *pGroup, + const bool save); + +static int _tracker_mem_add_storage(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail **ppStorageServer, const char *id, \ + const char *ip_addr, const bool bNeedSleep, \ + const bool bNeedLock, bool *bInserted); + +static int tracker_mem_add_storage(TrackerClientInfo *pClientInfo, \ + const char *id, const char *ip_addr, \ + const bool bNeedSleep, const bool bNeedLock, bool *bInserted); + +static int tracker_mem_add_group_ex(FDFSGroups *pGroups, \ + TrackerClientInfo *pClientInfo, const char *group_name, \ + const bool bNeedSleep, bool *bInserted); + +static int tracker_mem_destroy_groups(FDFSGroups *pGroups, const bool saveFiles); + +char *g_tracker_sys_filenames[TRACKER_SYS_FILE_COUNT] = { + STORAGE_GROUPS_LIST_FILENAME_NEW, + STORAGE_SERVERS_LIST_FILENAME_NEW, + STORAGE_SYNC_TIMESTAMP_FILENAME, + STORAGE_SERVERS_CHANGELOG_FILENAME +}; + +#define TRACKER_CHOWN(path, current_uid, current_gid) \ + if (!(g_run_by_gid == current_gid && g_run_by_uid == current_uid)) \ + { \ + if (chown(path, g_run_by_uid, g_run_by_gid) != 0) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "chown \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, path, \ + errno, STRERROR(errno)); \ + return errno != 0 ? errno : EPERM; \ + } \ + } + +#define TRACKER_FCHOWN(fd, path, current_uid, current_gid) \ + if (!(g_run_by_gid == current_gid && g_run_by_uid == current_uid)) \ + { \ + if (fchown(fd, g_run_by_uid, g_run_by_gid) != 0) \ + { \ + logError("file: "__FILE__", line: %d, " \ + "chown \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, path, \ + errno, STRERROR(errno)); \ + return errno != 0 ? errno : EPERM; \ + } \ + } + + +int tracker_mem_pthread_lock() +{ + int result; + if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int tracker_mem_pthread_unlock() +{ + int result; + if ((result=pthread_mutex_unlock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int tracker_mem_file_lock() +{ + int result; + if ((result=pthread_mutex_lock(&mem_file_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int tracker_mem_file_unlock() +{ + int result; + if ((result=pthread_mutex_unlock(&mem_file_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +static int tracker_write_to_changelog(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pStorage, const char *pArg) +{ + char buff[256]; + int len; + int result; + + tracker_mem_file_lock(); + + len = snprintf(buff, sizeof(buff), "%d %s %s %d %s\n", \ + (int)g_current_time, pGroup->group_name, pStorage->id, \ + pStorage->status, pArg != NULL ? pArg : ""); + + if (write(changelog_fd, buff, len) != len) + { + tracker_mem_file_unlock(); + + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "write to file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, STORAGE_SERVERS_CHANGELOG_FILENAME, \ + result, STRERROR(result)); + + return result; + } + + g_changelog_fsize += len; + result = fsync(changelog_fd); + if (result != 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "call fsync of file: %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, STORAGE_SERVERS_CHANGELOG_FILENAME, \ + result, STRERROR(result)); + } + + tracker_mem_file_unlock(); + + return result; +} + +static int tracker_malloc_storage_path_mbs(FDFSStorageDetail *pStorage, \ + const int store_path_count) +{ + int alloc_bytes; + + if (store_path_count <= 0) + { + return 0; + } + + alloc_bytes = sizeof(int64_t) * store_path_count; + pStorage->path_total_mbs = (int64_t *)malloc(alloc_bytes); + if (pStorage->path_total_mbs == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, alloc_bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + pStorage->path_free_mbs = (int64_t *)malloc(alloc_bytes); + if (pStorage->path_free_mbs == NULL) + { + free(pStorage->path_total_mbs); + pStorage->path_total_mbs = NULL; + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, alloc_bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + memset(pStorage->path_total_mbs, 0, alloc_bytes); + memset(pStorage->path_free_mbs, 0, alloc_bytes); + + return 0; +} + +static int tracker_realloc_storage_path_mbs(FDFSStorageDetail *pStorage, \ + const int old_store_path_count, const int new_store_path_count) +{ + int alloc_bytes; + int copy_bytes; + int64_t *new_path_total_mbs; + int64_t *new_path_free_mbs; + + if (new_store_path_count <= 0) + { + return EINVAL; + } + + alloc_bytes = sizeof(int64_t) * new_store_path_count; + + new_path_total_mbs = (int64_t *)malloc(alloc_bytes); + if (new_path_total_mbs == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, alloc_bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + new_path_free_mbs = (int64_t *)malloc(alloc_bytes); + if (new_path_free_mbs == NULL) + { + free(new_path_total_mbs); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, alloc_bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + memset(new_path_total_mbs, 0, alloc_bytes); + memset(new_path_free_mbs, 0, alloc_bytes); + + if (old_store_path_count == 0) + { + pStorage->path_total_mbs = new_path_total_mbs; + pStorage->path_free_mbs = new_path_free_mbs; + + return 0; + } + + copy_bytes = (old_store_path_count < new_store_path_count ? \ + old_store_path_count : new_store_path_count) * sizeof(int64_t); + memcpy(new_path_total_mbs, pStorage->path_total_mbs, copy_bytes); + memcpy(new_path_free_mbs, pStorage->path_free_mbs, copy_bytes); + + free(pStorage->path_total_mbs); + free(pStorage->path_free_mbs); + + pStorage->path_total_mbs = new_path_total_mbs; + pStorage->path_free_mbs = new_path_free_mbs; + + return 0; +} + +static int tracker_realloc_group_path_mbs(FDFSGroupInfo *pGroup, \ + const int new_store_path_count) +{ + FDFSStorageDetail **ppStorage; + FDFSStorageDetail **ppEnd; + int result; + + ppEnd = pGroup->all_servers + pGroup->alloc_size; + for (ppStorage=pGroup->all_servers; ppStoragestore_path_count, new_store_path_count)) != 0) + { + return result; + } + } + + pGroup->store_path_count = new_store_path_count; + + return 0; +} + +static int tracker_malloc_group_path_mbs(FDFSGroupInfo *pGroup) +{ + FDFSStorageDetail **ppStorage; + FDFSStorageDetail **ppEnd; + int result; + + ppEnd = pGroup->all_servers + pGroup->alloc_size; + for (ppStorage=pGroup->all_servers; ppStoragestore_path_count)) != 0) + { + return result; + } + } + + return 0; +} + +static int tracker_malloc_all_group_path_mbs(FDFSGroups *pGroups) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + int result; + + ppEnd = pGroups->groups + pGroups->alloc_size; + for (ppGroup=pGroups->groups; ppGroupstore_path_count == 0) + { + continue; + } + + if ((result=tracker_malloc_group_path_mbs(*ppGroup)) != 0) + { + return result; + } + } + + return 0; +} + +static int tracker_load_groups_old(FDFSGroups *pGroups, const char *data_path) +{ +#define STORAGE_DATA_GROUP_FIELDS 4 + + FILE *fp; + char szLine[256]; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *fields[STORAGE_DATA_GROUP_FIELDS]; + int result; + int col_count; + TrackerClientInfo clientInfo; + bool bInserted; + + if ((fp=fopen(STORAGE_GROUPS_LIST_FILENAME_OLD, "r")) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s/%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, STORAGE_GROUPS_LIST_FILENAME_OLD, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + result = 0; + while (fgets(szLine, sizeof(szLine), fp) != NULL) + { + if (*szLine == '\0') + { + continue; + } + + col_count = splitEx(szLine, STORAGE_DATA_FIELD_SEPERATOR, \ + fields, STORAGE_DATA_GROUP_FIELDS); + if (col_count != STORAGE_DATA_GROUP_FIELDS && \ + col_count != STORAGE_DATA_GROUP_FIELDS - 2) + { + logError("file: "__FILE__", line: %d, " \ + "the format of the file \"%s/%s\" is invalid", \ + __LINE__, data_path, \ + STORAGE_GROUPS_LIST_FILENAME_OLD); + result = errno != 0 ? errno : EINVAL; + break; + } + + memset(&clientInfo, 0, sizeof(TrackerClientInfo)); + snprintf(group_name, sizeof(group_name),\ + "%s", trim(fields[0])); + if ((result=tracker_mem_add_group_ex(pGroups, &clientInfo, \ + group_name, false, &bInserted)) != 0) + { + break; + } + + if (!bInserted) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "group \"%s\" is duplicate", \ + __LINE__, data_path, \ + STORAGE_GROUPS_LIST_FILENAME_OLD, \ + group_name); + result = errno != 0 ? errno : EEXIST; + break; + } + + clientInfo.pGroup->storage_port = atoi(trim(fields[1])); + if (col_count == STORAGE_DATA_GROUP_FIELDS - 2) + { //version < V1.12 + clientInfo.pGroup->store_path_count = 0; + clientInfo.pGroup->subdir_count_per_path = 0; + } + else + { + clientInfo.pGroup->store_path_count = \ + atoi(trim(fields[2])); + clientInfo.pGroup->subdir_count_per_path = \ + atoi(trim(fields[3])); + } + } + + fclose(fp); + return result; +} + +static int tracker_mem_get_storage_id(const char *group_name, \ + const char *pIpAddr, char *storage_id) +{ + FDFSStorageIdInfo *pStorageIdInfo; + pStorageIdInfo = fdfs_get_storage_id_by_ip(group_name, pIpAddr); + if (pStorageIdInfo == NULL) + { + return ENOENT; + } + + strcpy(storage_id, pStorageIdInfo->id); + return 0; +} + +static int tracker_load_groups_new(FDFSGroups *pGroups, const char *data_path, + FDFSStorageSync **ppTrunkServers, int *nTrunkServerCount) +{ + IniContext iniContext; + FDFSGroupInfo *pGroup; + char *group_name; + char *pValue; + int nStorageSyncSize; + int group_count; + int result; + int i; + char section_name[64]; + TrackerClientInfo clientInfo; + bool bInserted; + + *nTrunkServerCount = 0; + *ppTrunkServers = NULL; + if (!fileExists(STORAGE_GROUPS_LIST_FILENAME_NEW) && \ + fileExists(STORAGE_GROUPS_LIST_FILENAME_OLD)) + { + logDebug("file: "__FILE__", line: %d, " \ + "convert old data file %s to new data file %s", \ + __LINE__, STORAGE_GROUPS_LIST_FILENAME_OLD, \ + STORAGE_GROUPS_LIST_FILENAME_NEW); + if ((result=tracker_load_groups_old(pGroups, data_path)) == 0) + { + if ((result=tracker_save_groups()) == 0) + { + unlink(STORAGE_GROUPS_LIST_FILENAME_OLD); + } + } + + return result; + } + + if ((result=iniLoadFromFile(STORAGE_GROUPS_LIST_FILENAME_NEW, \ + &iniContext)) != 0) + { + return result; + } + + group_count = iniGetIntValue(GROUP_SECTION_NAME_GLOBAL, \ + GROUP_ITEM_GROUP_COUNT, \ + &iniContext, -1); + if (group_count < 0) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "item \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_GROUPS_LIST_FILENAME_NEW, \ + GROUP_ITEM_GROUP_COUNT); + return ENOENT; + } + + nStorageSyncSize = 0; + for (i=1; i<=group_count; i++) + { + sprintf(section_name, "%s"GROUP_SECTION_NO_FORMAT, \ + GROUP_SECTION_NAME_PREFIX, i); + + group_name = iniGetStrValue(section_name, \ + GROUP_ITEM_GROUP_NAME, &iniContext); + if (group_name == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "item \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_GROUPS_LIST_FILENAME_NEW, \ + GROUP_ITEM_GROUP_NAME); + result = ENOENT; + break; + } + + memset(&clientInfo, 0, sizeof(TrackerClientInfo)); + if ((result=tracker_mem_add_group_ex(pGroups, &clientInfo, \ + group_name, false, &bInserted)) != 0) + { + break; + } + + if (!bInserted) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "group \"%s\" is duplicate", \ + __LINE__, data_path, \ + STORAGE_GROUPS_LIST_FILENAME_NEW, \ + group_name); + result = errno != 0 ? errno : EEXIST; + break; + } + + pGroup = clientInfo.pGroup; + pGroup->storage_port = iniGetIntValue(section_name, \ + GROUP_ITEM_STORAGE_PORT, &iniContext, 0); + pGroup->storage_http_port = iniGetIntValue(section_name, \ + GROUP_ITEM_STORAGE_HTTP_PORT, &iniContext, 0); + pGroup->store_path_count = iniGetIntValue(section_name, \ + GROUP_ITEM_STORE_PATH_COUNT, &iniContext, 0); + pGroup->subdir_count_per_path = iniGetIntValue(section_name, \ + GROUP_ITEM_SUBDIR_COUNT_PER_PATH, &iniContext, 0); + pGroup->current_trunk_file_id = iniGetIntValue(section_name, \ + GROUP_ITEM_CURRENT_TRUNK_FILE_ID, &iniContext, 0); + pValue = iniGetStrValue(section_name, \ + GROUP_ITEM_LAST_TRUNK_SERVER, &iniContext); + if (pValue != NULL) + { + snprintf(pGroup->last_trunk_server_id, + sizeof(pGroup->last_trunk_server_id), \ + "%s", pValue); + if (g_use_storage_id && (*pValue != '\0' && \ + !fdfs_is_server_id_valid(pValue))) + { + if (tracker_mem_get_storage_id( \ + pGroup->group_name, pValue, \ + pGroup->last_trunk_server_id) != 0) + { + logWarning("file: "__FILE__", line: %d,"\ + "server id of group name: %s " \ + "and last trunk ip address: %s"\ + " NOT exist", __LINE__, \ + pGroup->group_name, pValue); + *pGroup->last_trunk_server_id = '\0'; + } + } + } + + pValue = iniGetStrValue(section_name, \ + GROUP_ITEM_TRUNK_SERVER, &iniContext); + if (pValue != NULL && *pValue != '\0') + { + if (nStorageSyncSize <= *nTrunkServerCount) + { + nStorageSyncSize += 8; + *ppTrunkServers = (FDFSStorageSync *)realloc( \ + *ppTrunkServers, \ + sizeof(FDFSStorageSync) * nStorageSyncSize); + if (*ppTrunkServers == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "realloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageSync) * \ + nStorageSyncSize); + break; + } + } + + strcpy((*ppTrunkServers)[*nTrunkServerCount].group_name, \ + clientInfo.pGroup->group_name); + snprintf((*ppTrunkServers)[*nTrunkServerCount].id, \ + FDFS_STORAGE_ID_MAX_SIZE, "%s", pValue); + if (g_use_storage_id && !fdfs_is_server_id_valid(pValue)) + { + if ((result=tracker_mem_get_storage_id( \ + clientInfo.pGroup->group_name, pValue, \ + (*ppTrunkServers)[*nTrunkServerCount].id)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server id of group name: %s and " \ + "trunk server ip address: %s NOT " \ + "exist", __LINE__, \ + clientInfo.pGroup->group_name, pValue); + break; + } + } + + (*nTrunkServerCount)++; + } + } + + iniFreeContext(&iniContext); + + return result; +} + +static int tracker_locate_group_trunk_servers(FDFSGroups *pGroups, \ + FDFSStorageSync *pTrunkServers, \ + const int nTrunkServerCount, const bool bLoadFromFile) +{ + FDFSGroupInfo *pGroup; + FDFSStorageDetail *pStorage; + FDFSStorageSync *pServer; + FDFSStorageSync *pTrunkEnd; + + if (nTrunkServerCount == 0) + { + return 0; + } + + pTrunkEnd = pTrunkServers + nTrunkServerCount; + for (pServer=pTrunkServers; pServergroup_name); + if (pGroup == NULL) + { + continue; + } + + pStorage = tracker_mem_get_storage(pGroup, pServer->id); + if (pStorage == NULL) + { + char buff[MAX_PATH_SIZE+64]; + if (bLoadFromFile) + { + snprintf(buff, sizeof(buff), \ + "in the file \"%s/data/%s\", ", \ + g_fdfs_base_path, \ + STORAGE_GROUPS_LIST_FILENAME_NEW); + } + else + { + *buff = '\0'; + } + + logError("file: "__FILE__", line: %d, " \ + "%sgroup_name: %s, trunk server \"%s\" " \ + "does not exist", __LINE__, buff, \ + pServer->group_name, pServer->id); + + return ENOENT; + } + + pGroup->pTrunkServer = pStorage; + pGroup->trunk_chg_count++; + if (*(pGroup->last_trunk_server_id) == '\0') + { + strcpy(pGroup->last_trunk_server_id, pStorage->id); + } + } + + return 0; +} + +static int tracker_locate_storage_sync_server(FDFSGroups *pGroups, \ + FDFSStorageSync *pStorageSyncs, \ + const int nStorageSyncCount, const bool bLoadFromFile) +{ + FDFSGroupInfo *pGroup; + FDFSStorageDetail *pStorage; + FDFSStorageSync *pSyncServer; + FDFSStorageSync *pSyncEnd; + + if (nStorageSyncCount == 0) + { + return 0; + } + + pSyncEnd = pStorageSyncs + nStorageSyncCount; + for (pSyncServer=pStorageSyncs; pSyncServergroup_name); + if (pGroup == NULL) + { + continue; + } + + pStorage=tracker_mem_get_storage(pGroup, pSyncServer->id); + if (pStorage == NULL) + { + continue; + } + + pStorage->psync_src_server = tracker_mem_get_storage(pGroup, \ + pSyncServer->sync_src_id); + if (pStorage->psync_src_server == NULL) + { + char buff[MAX_PATH_SIZE+64]; + if (bLoadFromFile) + { + snprintf(buff, sizeof(buff), \ + "in the file \"%s/data/%s\", ", \ + g_fdfs_base_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW); + } + else + { + *buff = '\0'; + } + + logError("file: "__FILE__", line: %d, " \ + "%sgroup_name: %s, storage server \"%s\" " \ + "does not exist", __LINE__, buff, \ + pSyncServer->group_name, \ + pSyncServer->sync_src_id); + + return ENOENT; + } + } + + return 0; +} + +static int tracker_load_storages_old(FDFSGroups *pGroups, const char *data_path) +{ +#define STORAGE_DATA_SERVER_FIELDS 22 + + FILE *fp; + char szLine[256]; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *fields[STORAGE_DATA_SERVER_FIELDS]; + char ip_addr[IP_ADDRESS_SIZE]; + char *psync_src_id; + FDFSStorageDetail *pStorage; + FDFSStorageSync *pStorageSyncs; + int nStorageSyncSize; + int nStorageSyncCount; + int cols; + int result; + TrackerClientInfo clientInfo; + bool bInserted; + + if ((fp=fopen(STORAGE_SERVERS_LIST_FILENAME_OLD, "r")) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s/%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, STORAGE_SERVERS_LIST_FILENAME_OLD, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + nStorageSyncSize = 0; + nStorageSyncCount = 0; + pStorageSyncs = NULL; + result = 0; + while (fgets(szLine, sizeof(szLine), fp) != NULL) + { + if (*szLine == '\0') + { + continue; + } + + cols = splitEx(szLine, STORAGE_DATA_FIELD_SEPERATOR, \ + fields, STORAGE_DATA_SERVER_FIELDS); + if (cols != STORAGE_DATA_SERVER_FIELDS && \ + cols != STORAGE_DATA_SERVER_FIELDS - 2 && \ + cols != STORAGE_DATA_SERVER_FIELDS - 4 && \ + cols != STORAGE_DATA_SERVER_FIELDS - 5) + { + logError("file: "__FILE__", line: %d, " \ + "the format of the file \"%s/%s\" is invalid" \ + ", colums: %d != expect colums: " \ + "%d or %d or %d or %d", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_OLD, \ + cols, STORAGE_DATA_SERVER_FIELDS, \ + STORAGE_DATA_SERVER_FIELDS - 2, \ + STORAGE_DATA_SERVER_FIELDS - 4, \ + STORAGE_DATA_SERVER_FIELDS - 5); + result = EINVAL; + break; + } + + memset(&clientInfo, 0, sizeof(TrackerClientInfo)); + snprintf(group_name, sizeof(group_name), "%s", trim(fields[0])); + snprintf(ip_addr, sizeof(ip_addr), "%s", trim(fields[1])); + if ((clientInfo.pGroup=tracker_mem_get_group_ex(pGroups, \ + group_name)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "group \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_OLD, \ + group_name); + result = errno != 0 ? errno : ENOENT; + break; + } + + if ((result=tracker_mem_add_storage(&clientInfo, NULL, ip_addr, \ + false, false, &bInserted)) != 0) + { + break; + } + + if (!bInserted) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "storage \"%s\" is duplicate", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_OLD, ip_addr); + result = errno != 0 ? errno : EEXIST; + break; + } + + pStorage = clientInfo.pStorage; + pStorage->status = atoi(trim_left(fields[2])); + if (!((pStorage->status == \ + FDFS_STORAGE_STATUS_WAIT_SYNC) || \ + (pStorage->status == \ + FDFS_STORAGE_STATUS_SYNCING) || \ + (pStorage->status == \ + FDFS_STORAGE_STATUS_INIT))) + { + pStorage->status = \ + FDFS_STORAGE_STATUS_OFFLINE; + } + + psync_src_id = trim(fields[3]); + pStorage->sync_until_timestamp = atoi( \ + trim_left(fields[4])); + pStorage->stat.total_upload_count = strtoll( \ + trim_left(fields[5]), NULL, 10); + pStorage->stat.success_upload_count = strtoll( \ + trim_left(fields[6]), NULL, 10); + pStorage->stat.total_set_meta_count = strtoll( \ + trim_left(fields[7]), NULL, 10); + pStorage->stat.success_set_meta_count = strtoll( \ + trim_left(fields[8]), NULL, 10); + pStorage->stat.total_delete_count = strtoll( \ + trim_left(fields[9]), NULL, 10); + pStorage->stat.success_delete_count = strtoll( \ + trim_left(fields[10]), NULL, 10); + pStorage->stat.total_download_count = strtoll( \ + trim_left(fields[11]), NULL, 10); + pStorage->stat.success_download_count = strtoll( \ + trim_left(fields[12]), NULL, 10); + pStorage->stat.total_get_meta_count = strtoll( \ + trim_left(fields[13]), NULL, 10); + pStorage->stat.success_get_meta_count = strtoll( \ + trim_left(fields[14]), NULL, 10); + pStorage->stat.last_source_update = atoi( \ + trim_left(fields[15])); + pStorage->stat.last_sync_update = atoi( \ + trim_left(fields[16])); + if (cols > STORAGE_DATA_SERVER_FIELDS - 5) + { + pStorage->changelog_offset = strtoll( \ + trim_left(fields[17]), NULL, 10); + if (pStorage->changelog_offset < 0) + { + pStorage->changelog_offset = 0; + } + if (pStorage->changelog_offset > \ + g_changelog_fsize) + { + pStorage->changelog_offset = \ + g_changelog_fsize; + } + + if (cols > STORAGE_DATA_SERVER_FIELDS - 4) + { + pStorage->storage_port = \ + atoi(trim_left(fields[18])); + pStorage->storage_http_port = \ + atoi(trim_left(fields[19])); + if (cols > STORAGE_DATA_SERVER_FIELDS - 2) + { + pStorage->join_time = \ + (time_t)atoi(trim_left(fields[20])); + + snprintf(pStorage->version, \ + sizeof(pStorage->version), + "%s", trim(fields[21])); + } + } + } + + if (*psync_src_id == '\0') + { + continue; + } + + if (nStorageSyncSize <= nStorageSyncCount) + { + nStorageSyncSize += 8; + pStorageSyncs = (FDFSStorageSync *)realloc( \ + pStorageSyncs, \ + sizeof(FDFSStorageSync) * nStorageSyncSize); + if (pStorageSyncs == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "realloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageSync) * \ + nStorageSyncSize); + break; + } + } + + strcpy(pStorageSyncs[nStorageSyncCount].group_name, \ + clientInfo.pGroup->group_name); + strcpy(pStorageSyncs[nStorageSyncCount].id, pStorage->id); + snprintf(pStorageSyncs[nStorageSyncCount].sync_src_id, \ + FDFS_STORAGE_ID_MAX_SIZE, "%s", psync_src_id); + + nStorageSyncCount++; + + } + + fclose(fp); + + if (pStorageSyncs == NULL) + { + return result; + } + + if (result != 0) + { + free(pStorageSyncs); + return result; + } + + result = tracker_locate_storage_sync_server(pGroups, pStorageSyncs, \ + nStorageSyncCount, true); + free(pStorageSyncs); + return result; +} + +static int tracker_load_storages_new(FDFSGroups *pGroups, const char *data_path) +{ + IniContext iniContext; + char *group_name; + char *storage_id; + char *ip_addr; + char *psync_src_id; + char *pValue; + FDFSStorageDetail *pStorage; + FDFSStorageStat *pStat; + FDFSStorageSync *pStorageSyncs; + int nStorageSyncSize; + int nStorageSyncCount; + int storage_count; + int i; + int result; + char section_name[64]; + TrackerClientInfo clientInfo; + bool bInserted; + + if (!fileExists(STORAGE_SERVERS_LIST_FILENAME_NEW) && \ + fileExists(STORAGE_SERVERS_LIST_FILENAME_OLD)) + { + logDebug("file: "__FILE__", line: %d, " \ + "convert old data file %s to new data file %s", \ + __LINE__, STORAGE_SERVERS_LIST_FILENAME_OLD, \ + STORAGE_SERVERS_LIST_FILENAME_NEW); + if ((result=tracker_load_storages_old(pGroups, data_path)) == 0) + { + if ((result=tracker_save_storages()) == 0) + { + unlink(STORAGE_SERVERS_LIST_FILENAME_OLD); + } + } + + return result; + } + + if ((result=iniLoadFromFile(STORAGE_SERVERS_LIST_FILENAME_NEW, \ + &iniContext)) != 0) + { + return result; + } + + storage_count = iniGetIntValue(STORAGE_SECTION_NAME_GLOBAL, \ + STORAGE_ITEM_STORAGE_COUNT, \ + &iniContext, -1); + if (storage_count < 0) + { + iniFreeContext(&iniContext); + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "item \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW, \ + STORAGE_ITEM_STORAGE_COUNT); + return ENOENT; + } + + nStorageSyncSize = 0; + nStorageSyncCount = 0; + pStorageSyncs = NULL; + result = 0; + for (i=1; i<=storage_count; i++) + { + sprintf(section_name, "%s"STORAGE_SECTION_NO_FORMAT, \ + STORAGE_SECTION_NAME_PREFIX, i); + + group_name = iniGetStrValue(section_name, \ + STORAGE_ITEM_GROUP_NAME, &iniContext); + if (group_name == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "item \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW, \ + STORAGE_ITEM_GROUP_NAME); + result = ENOENT; + break; + } + + storage_id = iniGetStrValue(section_name, \ + STORAGE_ITEM_SERVER_ID, &iniContext); + + ip_addr = iniGetStrValue(section_name, \ + STORAGE_ITEM_IP_ADDR, &iniContext); + if (ip_addr == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "item \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW, \ + STORAGE_ITEM_IP_ADDR); + result = ENOENT; + break; + } + if (*ip_addr == '\0') + { + logWarning("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "item \"%s\" is empty", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW, \ + STORAGE_ITEM_IP_ADDR); + continue; + } + + memset(&clientInfo, 0, sizeof(TrackerClientInfo)); + if ((clientInfo.pGroup=tracker_mem_get_group_ex(pGroups, \ + group_name)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "group \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW, \ + group_name); + result = errno != 0 ? errno : ENOENT; + break; + } + + if ((result=tracker_mem_add_storage(&clientInfo, storage_id, \ + ip_addr, false, false, &bInserted)) != 0) + { + break; + } + + if (!bInserted) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "storage \"%s\" is duplicate", \ + __LINE__, data_path, \ + STORAGE_SERVERS_LIST_FILENAME_NEW, ip_addr); + result = errno != 0 ? errno : EEXIST; + break; + } + + pStorage = clientInfo.pStorage; + pStat = &(pStorage->stat); + pStorage->status = iniGetIntValue(section_name, \ + STORAGE_ITEM_STATUS, &iniContext, 0); + + pValue = iniGetStrValue(section_name, \ + STORAGE_ITEM_DOMAIN_NAME, &iniContext); + if (pValue != NULL) + { + snprintf(pStorage->domain_name, \ + sizeof(pStorage->domain_name), "%s", pValue); + } + + pValue = iniGetStrValue(section_name, \ + STORAGE_ITEM_VERSION, &iniContext); + if (pValue != NULL) + { + snprintf(pStorage->version, \ + sizeof(pStorage->version), "%s", pValue); + } + + if (!((pStorage->status == \ + FDFS_STORAGE_STATUS_WAIT_SYNC) || \ + (pStorage->status == \ + FDFS_STORAGE_STATUS_SYNCING) || \ + (pStorage->status == \ + FDFS_STORAGE_STATUS_INIT))) + { + pStorage->status = FDFS_STORAGE_STATUS_OFFLINE; + } + + psync_src_id = iniGetStrValue(section_name, \ + STORAGE_ITEM_SYNC_SRC_SERVER, &iniContext); + if (psync_src_id == NULL) + { + psync_src_id = ""; + } + + pStorage->sync_until_timestamp = iniGetIntValue(section_name, \ + STORAGE_ITEM_SYNC_UNTIL_TIMESTAMP, &iniContext, 0); + pStorage->join_time = iniGetIntValue(section_name, \ + STORAGE_ITEM_JOIN_TIME, &iniContext, 0); + pStorage->total_mb = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_MB, &iniContext, 0); + pStorage->free_mb = iniGetInt64Value(section_name, \ + STORAGE_ITEM_FREE_MB, &iniContext, 0); + pStorage->store_path_count = iniGetIntValue(section_name, \ + STORAGE_ITEM_STORE_PATH_COUNT, &iniContext, 0); + pStorage->subdir_count_per_path = iniGetIntValue(section_name, \ + STORAGE_ITEM_SUBDIR_COUNT_PER_PATH, &iniContext, 0); + pStorage->upload_priority = iniGetIntValue(section_name, \ + STORAGE_ITEM_UPLOAD_PRIORITY, &iniContext, 0); + pStorage->storage_port = iniGetIntValue(section_name, \ + STORAGE_ITEM_STORAGE_PORT, &iniContext, 0); + pStorage->storage_http_port = iniGetIntValue(section_name, \ + STORAGE_ITEM_STORAGE_HTTP_PORT, &iniContext, 0); + pStat->total_upload_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_UPLOAD_COUNT, &iniContext, 0); + pStat->success_upload_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_UPLOAD_COUNT, &iniContext, 0); + pStat->total_append_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_APPEND_COUNT, &iniContext, 0); + pStat->success_append_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_APPEND_COUNT, &iniContext, 0); + pStat->total_set_meta_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_SET_META_COUNT, &iniContext, 0); + pStat->success_set_meta_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_SET_META_COUNT, &iniContext, 0); + pStat->total_delete_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_DELETE_COUNT, &iniContext, 0); + pStat->success_delete_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_DELETE_COUNT, &iniContext, 0); + pStat->total_download_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_DOWNLOAD_COUNT, &iniContext, 0); + pStat->success_download_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_DOWNLOAD_COUNT, &iniContext, 0); + pStat->total_get_meta_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_GET_META_COUNT, &iniContext, 0); + pStat->success_get_meta_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_GET_META_COUNT, &iniContext, 0); + pStat->total_create_link_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_CREATE_LINK_COUNT, &iniContext, 0); + pStat->success_create_link_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_CREATE_LINK_COUNT, &iniContext, 0); + pStat->total_delete_link_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_DELETE_LINK_COUNT, &iniContext, 0); + pStat->success_delete_link_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_DELETE_LINK_COUNT, &iniContext, 0); + pStat->total_upload_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_UPLOAD_BYTES, &iniContext, 0); + pStat->success_upload_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_UPLOAD_BYTES, &iniContext, 0); + pStat->total_append_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_APPEND_BYTES, &iniContext, 0); + pStat->success_append_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_APPEND_BYTES, &iniContext, 0); + pStat->total_download_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_DOWNLOAD_BYTES, &iniContext, 0); + pStat->success_download_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_DOWNLOAD_BYTES, &iniContext, 0); + pStat->total_sync_in_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_SYNC_IN_BYTES, &iniContext, 0); + pStat->success_sync_in_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_SYNC_IN_BYTES, &iniContext, 0); + pStat->total_sync_out_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_SYNC_OUT_BYTES, &iniContext, 0); + pStat->success_sync_out_bytes = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_SYNC_OUT_BYTES, &iniContext, 0); + pStat->total_file_open_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_FILE_OPEN_COUNT, &iniContext, 0); + pStat->success_file_open_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_FILE_OPEN_COUNT, &iniContext, 0); + pStat->total_file_read_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_FILE_READ_COUNT, &iniContext, 0); + pStat->success_file_read_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_FILE_READ_COUNT, &iniContext, 0); + pStat->total_file_write_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_TOTAL_FILE_WRITE_COUNT, &iniContext, 0); + pStat->success_file_write_count = iniGetInt64Value(section_name, \ + STORAGE_ITEM_SUCCESS_FILE_WRITE_COUNT, &iniContext, 0); + pStat->last_source_update = iniGetIntValue(section_name, \ + STORAGE_ITEM_LAST_SOURCE_UPDATE, &iniContext, 0); + pStat->last_sync_update = iniGetIntValue(section_name, \ + STORAGE_ITEM_LAST_SYNC_UPDATE, &iniContext, 0); + pStat->last_synced_timestamp = iniGetIntValue(section_name, \ + STORAGE_ITEM_LAST_SYNCED_TIMESTAMP, &iniContext, 0); + pStat->last_heart_beat_time = iniGetIntValue(section_name, \ + STORAGE_ITEM_LAST_HEART_BEAT_TIME, &iniContext, 0); + pStorage->changelog_offset = iniGetInt64Value(section_name, \ + STORAGE_ITEM_CHANGELOG_OFFSET, &iniContext, 0); + + if (*psync_src_id == '\0') + { + continue; + } + + if (nStorageSyncSize <= nStorageSyncCount) + { + if (nStorageSyncSize == 0) + { + nStorageSyncSize = 8; + } + else + { + nStorageSyncSize *= 2; + } + pStorageSyncs = (FDFSStorageSync *)realloc( \ + pStorageSyncs, \ + sizeof(FDFSStorageSync) * nStorageSyncSize); + if (pStorageSyncs == NULL) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "realloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageSync) * \ + nStorageSyncSize); + break; + } + } + + strcpy(pStorageSyncs[nStorageSyncCount].group_name, \ + clientInfo.pGroup->group_name); + strcpy(pStorageSyncs[nStorageSyncCount].id, pStorage->id); + snprintf(pStorageSyncs[nStorageSyncCount].sync_src_id, \ + FDFS_STORAGE_ID_MAX_SIZE, "%s", psync_src_id); + if (g_use_storage_id && !fdfs_is_server_id_valid( \ + psync_src_id)) + { + if ((result=tracker_mem_get_storage_id( \ + clientInfo.pGroup->group_name, psync_src_id, \ + pStorageSyncs[nStorageSyncCount].sync_src_id))\ + != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server id of group name: %s and " \ + "src storage ip address: %s NOT " \ + "exist", __LINE__, \ + clientInfo.pGroup->group_name, \ + psync_src_id); + break; + } + } + + nStorageSyncCount++; + } + + iniFreeContext(&iniContext); + + if (pStorageSyncs == NULL) + { + return result; + } + + if (result != 0) + { + free(pStorageSyncs); + return result; + } + + result = tracker_locate_storage_sync_server(pGroups, pStorageSyncs, \ + nStorageSyncCount, true); + free(pStorageSyncs); + return result; +} + +static int tracker_load_sync_timestamps(FDFSGroups *pGroups, const char *data_path) +{ +#define STORAGE_SYNC_TIME_MAX_FIELDS 2 + FDFS_MAX_SERVERS_EACH_GROUP + + FILE *fp; + char szLine[512]; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char previous_group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char src_storage_id[FDFS_STORAGE_ID_MAX_SIZE]; + char *fields[STORAGE_SYNC_TIME_MAX_FIELDS]; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + FDFSGroupInfo *pGroup; + int cols; + int src_index; + int dest_index; + int curr_synced_timestamp; + int result; + + if (!fileExists(STORAGE_SYNC_TIMESTAMP_FILENAME)) + { + return 0; + } + + if ((fp=fopen(STORAGE_SYNC_TIMESTAMP_FILENAME, "r")) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "open file \"%s/%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, STORAGE_SYNC_TIMESTAMP_FILENAME, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + pGroup = NULL; + src_index = 0; + *previous_group_name = '\0'; + result = 0; + while (fgets(szLine, sizeof(szLine), fp) != NULL) + { + if (*szLine == '\0' || *szLine == '\n') + { + continue; + } + + if ((cols=splitEx(szLine, STORAGE_DATA_FIELD_SEPERATOR, \ + fields, STORAGE_SYNC_TIME_MAX_FIELDS)) <= 2) + { + logError("file: "__FILE__", line: %d, " \ + "the format of the file \"%s/%s\" is invalid" \ + ", colums: %d <= 2", \ + __LINE__, data_path, \ + STORAGE_SYNC_TIMESTAMP_FILENAME, cols); + result = errno != 0 ? errno : EINVAL; + break; + } + + snprintf(group_name, sizeof(group_name), \ + "%s", trim(fields[0])); + snprintf(src_storage_id, sizeof(src_storage_id), \ + "%s", trim(fields[1])); + if (strcmp(group_name, previous_group_name) != 0 || \ + pGroup == NULL) + { + if ((pGroup=tracker_mem_get_group_ex(pGroups, \ + group_name)) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "in the file \"%s/%s\", " \ + "group \"%s\" is not found", \ + __LINE__, data_path, \ + STORAGE_SYNC_TIMESTAMP_FILENAME, \ + group_name); + result = errno != 0 ? errno : ENOENT; + break; + } + + strcpy(previous_group_name, group_name); + src_index = 0; + } + + if (src_index >= pGroup->count) + { + logError("file: "__FILE__", line: %d, " \ + "the format of the file \"%s/%s\" is invalid" \ + ", group: %s, row count:%d > server count:%d",\ + __LINE__, data_path, \ + STORAGE_SYNC_TIMESTAMP_FILENAME, \ + group_name, src_index+1, pGroup->count); + result = errno != 0 ? errno : EINVAL; + break; + } + + if (g_use_storage_id && !fdfs_is_server_id_valid( \ + src_storage_id)) + { + if ((result=tracker_mem_get_storage_id( \ + group_name, src_storage_id, \ + src_storage_id)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server id of group name: %s and " \ + "storage ip address: %s NOT " \ + "exist", __LINE__, \ + group_name, src_storage_id); + break; + } + } + + if (strcmp(pGroup->all_servers[src_index]->id, \ + src_storage_id) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "in data file: \"%s/%s\", " \ + "group: %s, src server id: %s != %s",\ + __LINE__, data_path, \ + STORAGE_SYNC_TIMESTAMP_FILENAME, \ + group_name, src_storage_id, \ + pGroup->all_servers[src_index]->id); + result = errno != 0 ? errno : EINVAL; + break; + } + + if (cols > pGroup->count + 2) + { + logError("file: "__FILE__", line: %d, " \ + "the format of the file \"%s/%s\" is invalid" \ + ", group_name: %s, colums: %d > %d", \ + __LINE__, data_path, \ + STORAGE_SYNC_TIMESTAMP_FILENAME, \ + group_name, cols, pGroup->count + 2); + result = errno != 0 ? errno : EINVAL; + break; + } + + for (dest_index=0; dest_indexlast_sync_timestamps[src_index][dest_index] = \ + atoi(trim_left(fields[2 + dest_index])); + } + + src_index++; + } + + fclose(fp); + + if (result != 0) + { + return result; + } + + ppEnd = pGroups->groups + pGroups->count; + for (ppGroup=pGroups->groups; ppGroupcount <= 1) + { + continue; + } + + for (dest_index=0; dest_index<(*ppGroup)->count; dest_index++) + { + if (pGroups->store_server == FDFS_STORE_SERVER_ROUND_ROBIN) + { + int min_synced_timestamp; + + min_synced_timestamp = 0; + for (src_index=0; src_index<(*ppGroup)->count; \ + src_index++) + { + if (src_index == dest_index) + { + continue; + } + + curr_synced_timestamp = \ + (*ppGroup)->last_sync_timestamps \ + [src_index][dest_index]; + if (curr_synced_timestamp == 0) + { + continue; + } + + if (min_synced_timestamp == 0) + { + min_synced_timestamp = \ + curr_synced_timestamp; + } + else if (curr_synced_timestamp < \ + min_synced_timestamp) + { + min_synced_timestamp = \ + curr_synced_timestamp; + } + } + + (*ppGroup)->all_servers[dest_index]->stat. \ + last_synced_timestamp = min_synced_timestamp; + } + else + { + int max_synced_timestamp; + + max_synced_timestamp = 0; + for (src_index=0; src_index<(*ppGroup)->count; \ + src_index++) + { + if (src_index == dest_index) + { + continue; + } + + curr_synced_timestamp = \ + (*ppGroup)->last_sync_timestamps \ + [src_index][dest_index]; + if (curr_synced_timestamp > \ + max_synced_timestamp) + { + max_synced_timestamp = \ + curr_synced_timestamp; + } + } + + (*ppGroup)->all_servers[dest_index]->stat. \ + last_synced_timestamp = max_synced_timestamp; + } + } + } + + return result; +} + +static int tracker_load_data(FDFSGroups *pGroups) +{ + char data_path[MAX_PATH_SIZE]; + int result; + FDFSStorageSync *pTrunkServers; + int nTrunkServerCount; + + snprintf(data_path, sizeof(data_path), "%s/data", g_fdfs_base_path); + if (!fileExists(data_path)) + { + if (mkdir(data_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + TRACKER_CHOWN(data_path, geteuid(), getegid()) + } + + if (chdir(data_path) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "chdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + if (!fileExists(STORAGE_GROUPS_LIST_FILENAME_OLD) && \ + !fileExists(STORAGE_GROUPS_LIST_FILENAME_NEW)) + { + return 0; + } + + if ((result=tracker_load_groups_new(pGroups, data_path, \ + &pTrunkServers, &nTrunkServerCount)) != 0) + { + return result; + } + + if ((result=tracker_load_storages_new(pGroups, data_path)) != 0) + { + return result; + } + + if ((result=tracker_malloc_all_group_path_mbs(pGroups)) != 0) + { + return result; + } + + if ((result=tracker_load_sync_timestamps(pGroups, data_path)) != 0) + { + return result; + } + + if (g_if_use_trunk_file) + { + if ((result=tracker_locate_group_trunk_servers(pGroups, \ + pTrunkServers, nTrunkServerCount, true)) != 0) + { + return result; + } + } + + if (pTrunkServers != NULL) + { + free(pTrunkServers); + } + + return 0; +} + +int tracker_save_groups() +{ + char tmpFilename[MAX_PATH_SIZE]; + char trueFilename[MAX_PATH_SIZE]; + char buff[FDFS_GROUP_NAME_MAX_LEN + 256]; + int fd; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + int result; + int len; + + tracker_mem_file_lock(); + + snprintf(trueFilename, sizeof(trueFilename), "%s/data/%s", \ + g_fdfs_base_path, STORAGE_GROUPS_LIST_FILENAME_NEW); + snprintf(tmpFilename, sizeof(tmpFilename), "%s.tmp", trueFilename); + if ((fd=open(tmpFilename, O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0) + { + tracker_mem_file_unlock(); + + logError("file: "__FILE__", line: %d, " \ + "open \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + len = sprintf(buff, \ + "# global section\n" \ + "[%s]\n" \ + "\t%s=%d\n\n", \ + GROUP_SECTION_NAME_GLOBAL, \ + GROUP_ITEM_GROUP_COUNT, g_groups.count); + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + else + { + result = 0; + + ppEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; ppGroupgroup_name, \ + GROUP_SECTION_NAME_PREFIX, \ + (int)(ppGroup - g_groups.sorted_groups) + 1, \ + GROUP_ITEM_GROUP_NAME, \ + (*ppGroup)->group_name, \ + GROUP_ITEM_STORAGE_PORT, \ + (*ppGroup)->storage_port, \ + GROUP_ITEM_STORAGE_HTTP_PORT, \ + (*ppGroup)->storage_http_port, \ + GROUP_ITEM_STORE_PATH_COUNT, \ + (*ppGroup)->store_path_count, \ + GROUP_ITEM_SUBDIR_COUNT_PER_PATH, \ + (*ppGroup)->subdir_count_per_path, \ + GROUP_ITEM_CURRENT_TRUNK_FILE_ID, \ + (*ppGroup)->current_trunk_file_id, \ + GROUP_ITEM_TRUNK_SERVER, \ + (*ppGroup)->pTrunkServer != NULL ? \ + (*ppGroup)->pTrunkServer->id : "", + GROUP_ITEM_LAST_TRUNK_SERVER, \ + (*ppGroup)->last_trunk_server_id + ); + + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + break; + } + } + } + + if (result == 0) + { + if (fsync(fd) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "fsync file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + } + + close(fd); + + if (result == 0) + { + if (rename(tmpFilename, trueFilename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file \"%s\" to \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, trueFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + + TRACKER_CHOWN(trueFilename, geteuid(), getegid()) + } + + if (result != 0) + { + unlink(tmpFilename); + } + + tracker_mem_file_unlock(); + + return result; +} + +int tracker_save_storages() +{ + char tmpFilename[MAX_PATH_SIZE]; + char trueFilename[MAX_PATH_SIZE]; + char buff[4096]; + char id_buff[128]; + int fd; + int len; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + FDFSStorageDetail **ppStorage; + FDFSStorageDetail **ppStorageEnd; + FDFSStorageDetail *pStorage; + int result; + int count; + + tracker_mem_file_lock(); + + snprintf(trueFilename, sizeof(trueFilename), "%s/data/%s", \ + g_fdfs_base_path, STORAGE_SERVERS_LIST_FILENAME_NEW); + snprintf(tmpFilename, sizeof(tmpFilename), "%s.tmp", trueFilename); + if ((fd=open(tmpFilename, O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0) + { + tracker_mem_file_unlock(); + + logError("file: "__FILE__", line: %d, " \ + "open \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + *id_buff = '\0'; + count = 0; + result = 0; + ppGroupEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; \ + (ppGroup < ppGroupEnd) && (result == 0); ppGroup++) + { + ppStorageEnd = (*ppGroup)->all_servers + (*ppGroup)->count; + for (ppStorage=(*ppGroup)->all_servers; \ + ppStoragestatus == FDFS_STORAGE_STATUS_DELETED + || pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED) + { + continue; + } + + if (g_use_storage_id) + { + sprintf(id_buff, "\t%s=%s\n", \ + STORAGE_ITEM_SERVER_ID, pStorage->id); + } + + count++; + len = sprintf(buff, \ + "# storage %s:%d\n" \ + "[%s"STORAGE_SECTION_NO_FORMAT"]\n" \ + "%s" \ + "\t%s=%s\n" \ + "\t%s=%s\n" \ + "\t%s=%d\n" \ + "\t%s=%s\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s=%s\n" \ + "\t%s=%s\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s=%d\n" \ + "\t%s="INT64_PRINTF_FORMAT"\n\n", \ + pStorage->ip_addr, pStorage->storage_port, \ + STORAGE_SECTION_NAME_PREFIX, count, id_buff, \ + STORAGE_ITEM_GROUP_NAME, \ + (*ppGroup)->group_name, \ + STORAGE_ITEM_IP_ADDR, pStorage->ip_addr, \ + STORAGE_ITEM_STATUS, pStorage->status, \ + STORAGE_ITEM_VERSION, pStorage->version, \ + STORAGE_ITEM_JOIN_TIME, \ + (int)pStorage->join_time, \ + STORAGE_ITEM_STORAGE_PORT, \ + pStorage->storage_port, \ + STORAGE_ITEM_STORAGE_HTTP_PORT, \ + pStorage->storage_http_port, \ + STORAGE_ITEM_DOMAIN_NAME, \ + pStorage->domain_name, \ + STORAGE_ITEM_SYNC_SRC_SERVER, \ + (pStorage->psync_src_server != NULL ? \ + pStorage->psync_src_server->id: ""), \ + STORAGE_ITEM_SYNC_UNTIL_TIMESTAMP, \ + (int)pStorage->sync_until_timestamp, \ + STORAGE_ITEM_STORE_PATH_COUNT, \ + pStorage->store_path_count, \ + STORAGE_ITEM_SUBDIR_COUNT_PER_PATH, \ + pStorage->subdir_count_per_path, \ + STORAGE_ITEM_UPLOAD_PRIORITY, \ + pStorage->upload_priority, \ + STORAGE_ITEM_TOTAL_MB, pStorage->total_mb, \ + STORAGE_ITEM_FREE_MB, pStorage->free_mb, \ + STORAGE_ITEM_TOTAL_UPLOAD_COUNT, \ + pStorage->stat.total_upload_count, \ + STORAGE_ITEM_SUCCESS_UPLOAD_COUNT, \ + pStorage->stat.success_upload_count, \ + STORAGE_ITEM_TOTAL_APPEND_COUNT, \ + pStorage->stat.total_append_count, \ + STORAGE_ITEM_SUCCESS_APPEND_COUNT, \ + pStorage->stat.success_append_count, \ + STORAGE_ITEM_TOTAL_SET_META_COUNT, \ + pStorage->stat.total_set_meta_count, \ + STORAGE_ITEM_SUCCESS_SET_META_COUNT, \ + pStorage->stat.success_set_meta_count, \ + STORAGE_ITEM_TOTAL_DELETE_COUNT, \ + pStorage->stat.total_delete_count, \ + STORAGE_ITEM_SUCCESS_DELETE_COUNT, \ + pStorage->stat.success_delete_count, \ + STORAGE_ITEM_TOTAL_DOWNLOAD_COUNT, \ + pStorage->stat.total_download_count, \ + STORAGE_ITEM_SUCCESS_DOWNLOAD_COUNT, \ + pStorage->stat.success_download_count, \ + STORAGE_ITEM_TOTAL_GET_META_COUNT, \ + pStorage->stat.total_get_meta_count, \ + STORAGE_ITEM_SUCCESS_GET_META_COUNT, \ + pStorage->stat.success_get_meta_count, \ + STORAGE_ITEM_TOTAL_CREATE_LINK_COUNT, \ + pStorage->stat.total_create_link_count, \ + STORAGE_ITEM_SUCCESS_CREATE_LINK_COUNT, \ + pStorage->stat.success_create_link_count, \ + STORAGE_ITEM_TOTAL_DELETE_LINK_COUNT, \ + pStorage->stat.total_delete_link_count, \ + STORAGE_ITEM_SUCCESS_DELETE_LINK_COUNT, \ + pStorage->stat.success_delete_link_count, \ + STORAGE_ITEM_TOTAL_UPLOAD_BYTES, \ + pStorage->stat.total_upload_bytes, \ + STORAGE_ITEM_SUCCESS_UPLOAD_BYTES, \ + pStorage->stat.success_upload_bytes, \ + STORAGE_ITEM_TOTAL_APPEND_BYTES, \ + pStorage->stat.total_append_bytes, \ + STORAGE_ITEM_SUCCESS_APPEND_BYTES, \ + pStorage->stat.success_append_bytes, \ + STORAGE_ITEM_TOTAL_DOWNLOAD_BYTES, \ + pStorage->stat.total_download_bytes, \ + STORAGE_ITEM_SUCCESS_DOWNLOAD_BYTES, \ + pStorage->stat.success_download_bytes, \ + STORAGE_ITEM_TOTAL_SYNC_IN_BYTES, \ + pStorage->stat.total_sync_in_bytes, \ + STORAGE_ITEM_SUCCESS_SYNC_IN_BYTES, \ + pStorage->stat.success_sync_in_bytes, \ + STORAGE_ITEM_TOTAL_SYNC_OUT_BYTES, \ + pStorage->stat.total_sync_out_bytes, \ + STORAGE_ITEM_SUCCESS_SYNC_OUT_BYTES, \ + pStorage->stat.success_sync_out_bytes, \ + STORAGE_ITEM_TOTAL_FILE_OPEN_COUNT, \ + pStorage->stat.total_file_open_count, \ + STORAGE_ITEM_SUCCESS_FILE_OPEN_COUNT, \ + pStorage->stat.success_file_open_count, \ + STORAGE_ITEM_TOTAL_FILE_READ_COUNT, \ + pStorage->stat.total_file_read_count, \ + STORAGE_ITEM_SUCCESS_FILE_READ_COUNT, \ + pStorage->stat.success_file_read_count, \ + STORAGE_ITEM_TOTAL_FILE_WRITE_COUNT, \ + pStorage->stat.total_file_write_count, \ + STORAGE_ITEM_SUCCESS_FILE_WRITE_COUNT, \ + pStorage->stat.success_file_write_count, \ + STORAGE_ITEM_LAST_SOURCE_UPDATE, \ + (int)(pStorage->stat.last_source_update), \ + STORAGE_ITEM_LAST_SYNC_UPDATE, \ + (int)(pStorage->stat.last_sync_update), \ + STORAGE_ITEM_LAST_SYNCED_TIMESTAMP, \ + (int)pStorage->stat.last_synced_timestamp, \ + STORAGE_ITEM_LAST_HEART_BEAT_TIME, \ + (int)pStorage->stat.last_heart_beat_time, \ + STORAGE_ITEM_CHANGELOG_OFFSET, \ + pStorage->changelog_offset \ + ); + + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + break; + } + } + } + + if (result == 0) + { + len = sprintf(buff, \ + "\n# global section\n" \ + "[%s]\n" \ + "\t%s=%d\n", \ + STORAGE_SECTION_NAME_GLOBAL, \ + STORAGE_ITEM_STORAGE_COUNT, count); + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + } + + if (result == 0) + { + if (fsync(fd) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "fsync file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + } + + close(fd); + + if (result == 0) + { + if (rename(tmpFilename, trueFilename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file \"%s\" to \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, trueFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + + TRACKER_CHOWN(trueFilename, geteuid(), getegid()) + } + + if (result != 0) + { + unlink(tmpFilename); + } + + tracker_mem_file_unlock(); + + return result; +} + +int tracker_save_sync_timestamps() +{ + char tmpFilename[MAX_PATH_SIZE]; + char trueFilename[MAX_PATH_SIZE]; + char buff[512]; + int fd; + int len; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + int **last_sync_timestamps; + int i; + int k; + int result; + + tracker_mem_file_lock(); + + snprintf(trueFilename, sizeof(trueFilename), "%s/data/%s", \ + g_fdfs_base_path, STORAGE_SYNC_TIMESTAMP_FILENAME); + snprintf(tmpFilename, sizeof(tmpFilename), "%s.tmp", trueFilename); + if ((fd=open(tmpFilename, O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0) + { + tracker_mem_file_unlock(); + + logError("file: "__FILE__", line: %d, " \ + "open \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + result = 0; + ppGroupEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; \ + (ppGroup < ppGroupEnd) && (result == 0); ppGroup++) + { + last_sync_timestamps = (*ppGroup)->last_sync_timestamps; + for (i=0; i<(*ppGroup)->count; i++) + { + if ((*ppGroup)->all_servers[i]->status == \ + FDFS_STORAGE_STATUS_DELETED \ + || (*ppGroup)->all_servers[i]->status == \ + FDFS_STORAGE_STATUS_IP_CHANGED) + { + continue; + } + + len = sprintf(buff, "%s%c%s", (*ppGroup)->group_name, \ + STORAGE_DATA_FIELD_SEPERATOR, \ + (*ppGroup)->all_servers[i]->id); + for (k=0; k<(*ppGroup)->count; k++) + { + if ((*ppGroup)->all_servers[k]->status == \ + FDFS_STORAGE_STATUS_DELETED \ + || (*ppGroup)->all_servers[k]->status == \ + FDFS_STORAGE_STATUS_IP_CHANGED) + { + continue; + } + + len += sprintf(buff + len, "%c%d", \ + STORAGE_DATA_FIELD_SEPERATOR, \ + last_sync_timestamps[i][k]); + } + *(buff + len) = '\n'; + len++; + + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + break; + } + } + } + + if (result == 0) + { + if (fsync(fd) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "fsync file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + } + + close(fd); + + if (result == 0) + { + if (rename(tmpFilename, trueFilename) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "rename file \"%s\" to \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, tmpFilename, trueFilename, \ + errno, STRERROR(errno)); + result = errno != 0 ? errno : EIO; + } + + TRACKER_CHOWN(trueFilename, geteuid(), getegid()) + } + + if (result != 0) + { + unlink(tmpFilename); + } + + tracker_mem_file_unlock(); + + return result; +} + +int tracker_save_sys_files() +{ + int result; + + if ((result=tracker_save_groups()) != 0) + { + return result; + } + + if ((result=tracker_save_storages()) != 0) + { + return result; + } + + return tracker_save_sync_timestamps(); +} + +static int tracker_open_changlog_file() +{ + char data_path[MAX_PATH_SIZE]; + char filename[MAX_PATH_SIZE]; + + snprintf(data_path, sizeof(data_path), "%s/data", g_fdfs_base_path); + if (!fileExists(data_path)) + { + if (mkdir(data_path, 0755) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "mkdir \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, data_path, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + TRACKER_CHOWN(data_path, geteuid(), getegid()) + } + + snprintf(filename, sizeof(filename), "%s/data/%s", \ + g_fdfs_base_path, STORAGE_SERVERS_CHANGELOG_FILENAME); + changelog_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (changelog_fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + g_changelog_fsize = lseek(changelog_fd, 0, SEEK_END); + if (g_changelog_fsize < 0) + { + logError("file: "__FILE__", line: %d, " \ + "lseek file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, filename, errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + + TRACKER_FCHOWN(changelog_fd, filename, geteuid(), getegid()) + + return 0; +} + +static int tracker_mem_init_groups(FDFSGroups *pGroups) +{ + int result; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + + pGroups->alloc_size = TRACKER_MEM_ALLOC_ONCE; + pGroups->count = 0; + pGroups->current_write_group = 0; + pGroups->pStoreGroup = NULL; + pGroups->groups = (FDFSGroupInfo **)malloc( \ + sizeof(FDFSGroupInfo *) * pGroups->alloc_size); + if (pGroups->groups == NULL) + { + logCrit("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail!", __LINE__, \ + (int)sizeof(FDFSGroupInfo *) * pGroups->alloc_size); + return errno != 0 ? errno : ENOMEM; + } + + memset(pGroups->groups, 0, \ + sizeof(FDFSGroupInfo *) * pGroups->alloc_size); + + ppGroupEnd = pGroups->groups + pGroups->alloc_size; + for (ppGroup=pGroups->groups; ppGroupsorted_groups = (FDFSGroupInfo **) \ + malloc(sizeof(FDFSGroupInfo *) * pGroups->alloc_size); + if (pGroups->sorted_groups == NULL) + { + free(pGroups->groups); + pGroups->groups = NULL; + + logCrit("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail!", __LINE__, \ + (int)sizeof(FDFSGroupInfo *) * pGroups->alloc_size); + return errno != 0 ? errno : ENOMEM; + } + + memset(pGroups->sorted_groups, 0, \ + sizeof(FDFSGroupInfo *) * pGroups->alloc_size); + + if ((result=tracker_load_data(pGroups)) != 0) + { + return result; + } + + return 0; +} + +int tracker_mem_init() +{ + int result; + + if ((result=init_pthread_lock(&mem_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&mem_file_lock)) != 0) + { + return result; + } + + if ((result=tracker_open_changlog_file()) != 0) + { + return result; + } + + return tracker_mem_init_groups(&g_groups); +} + +static void tracker_free_last_sync_timestamps(int **last_sync_timestamps, \ + const int alloc_size) +{ + int i; + + if (last_sync_timestamps != NULL) + { + for (i=0; isorted_servers != NULL) + { + free(pGroup->sorted_servers); + pGroup->sorted_servers = NULL; + } + + if (pGroup->active_servers != NULL) + { + free(pGroup->active_servers); + pGroup->active_servers = NULL; + } + + if (pGroup->all_servers != NULL) + { + tracker_mem_free_storages(pGroup->all_servers, \ + pGroup->alloc_size); + pGroup->all_servers = NULL; + } + +#ifdef WITH_HTTPD + if (g_http_check_interval > 0) + { + if (pGroup->http_servers != NULL) + { + free(pGroup->http_servers); + pGroup->http_servers = NULL; + } + } +#endif + + tracker_free_last_sync_timestamps(pGroup->last_sync_timestamps, \ + pGroup->alloc_size); + pGroup->last_sync_timestamps = NULL; +} + +static int tracker_mem_init_group(FDFSGroupInfo *pGroup) +{ + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + int err_no; + + pGroup->alloc_size = TRACKER_MEM_ALLOC_ONCE; + pGroup->count = 0; + pGroup->all_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + if (pGroup->all_servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + return errno != 0 ? errno : ENOMEM; + } + + memset(pGroup->all_servers, 0, \ + sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + ppServerEnd = pGroup->all_servers + pGroup->alloc_size; + for (ppServer=pGroup->all_servers; ppServersorted_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + if (pGroup->sorted_servers == NULL) + { + tracker_mem_free_group(pGroup); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + return errno != 0 ? errno : ENOMEM; + } + memset(pGroup->sorted_servers, 0, \ + sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + + pGroup->active_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + if (pGroup->active_servers == NULL) + { + tracker_mem_free_group(pGroup); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + return errno != 0 ? errno : ENOMEM; + } + memset(pGroup->active_servers, 0, \ + sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + +#ifdef WITH_HTTPD + if (g_http_check_interval <= 0) + { + pGroup->http_servers = pGroup->active_servers; + } + else + { + pGroup->http_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *)*pGroup->alloc_size); + if (pGroup->http_servers == NULL) + { + tracker_mem_free_group(pGroup); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageDetail *) * \ + pGroup->alloc_size); + return errno != 0 ? errno : ENOMEM; + } + memset(pGroup->http_servers, 0, \ + sizeof(FDFSStorageDetail *) * pGroup->alloc_size); + g_http_servers_dirty = true; + } +#endif + + pGroup->last_sync_timestamps = tracker_malloc_last_sync_timestamps( \ + pGroup->alloc_size, &err_no); + return err_no; +} + +static int tracker_mem_destroy_groups(FDFSGroups *pGroups, const bool saveFiles) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + int result; + + if (pGroups->groups == NULL) + { + result = 0; + } + else + { + if (saveFiles) + { + result = tracker_save_sys_files(); + } + else + { + result = 0; + } + + ppEnd = pGroups->groups + pGroups->count; + for (ppGroup=pGroups->groups; ppGroupsorted_groups != NULL) + { + free(pGroups->sorted_groups); + pGroups->sorted_groups = NULL; + } + + free(pGroups->groups); + pGroups->groups = NULL; + } + + return result; +} + +int tracker_mem_destroy() +{ + int result; + + result = tracker_mem_destroy_groups(&g_groups, true); + + if (changelog_fd >= 0) + { + close(changelog_fd); + changelog_fd = -1; + } + + if (pthread_mutex_destroy(&mem_thread_lock) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_destroy fail", \ + __LINE__); + } + + if (pthread_mutex_destroy(&mem_file_lock) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_destroy fail", \ + __LINE__); + } + + return result; +} + +static void tracker_mem_free_groups(FDFSGroupInfo **groups, const int count) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + + ppGroupEnd = groups + count; + for (ppGroup=groups; ppGroupalloc_size + TRACKER_MEM_ALLOC_ONCE; + new_groups = (FDFSGroupInfo **)malloc(sizeof(FDFSGroupInfo *) * new_size); + if (new_groups == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(FDFSGroupInfo *) * new_size); + return errno != 0 ? errno : ENOMEM; + } + memset(new_groups, 0, sizeof(FDFSGroupInfo *) * new_size); + + ppGroupEnd = new_groups + new_size; + for (ppGroup=new_groups+pGroups->count; ppGroupgroups, \ + sizeof(FDFSGroupInfo *) * pGroups->count); + + new_sorted_groups = (FDFSGroupInfo **)malloc( \ + sizeof(FDFSGroupInfo *) * new_size); + if (new_sorted_groups == NULL) + { + tracker_mem_free_groups(new_groups, new_size); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(FDFSGroupInfo *) * new_size); + return errno != 0 ? errno : ENOMEM; + } + + memset(new_sorted_groups, 0, sizeof(FDFSGroupInfo *) * new_size); + memcpy(new_sorted_groups, pGroups->sorted_groups, \ + sizeof(FDFSGroupInfo *) * pGroups->count); + + old_groups = pGroups->groups; + old_sorted_groups = pGroups->sorted_groups; + pGroups->alloc_size = new_size; + pGroups->groups = new_groups; + pGroups->sorted_groups = new_sorted_groups; + + if (bNeedSleep) + { + sleep(1); + } + + free(old_groups); + free(old_sorted_groups); + + return 0; +} + +int tracker_get_group_file_count(FDFSGroupInfo *pGroup) +{ + int count; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + + count = 0; + ppServerEnd = pGroup->all_servers + pGroup->count; + for (ppServer=pGroup->all_servers; ppServerstat.success_upload_count - \ + (*ppServer)->stat.success_delete_count; + } + + return count; +} + +int tracker_get_group_success_upload_count(FDFSGroupInfo *pGroup) +{ + int count; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + + count = 0; + ppServerEnd = pGroup->all_servers + pGroup->count; + for (ppServer=pGroup->all_servers; ppServerstat.success_upload_count; + } + + return count; +} + +FDFSStorageDetail *tracker_get_group_sync_src_server(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pDestServer) +{ + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + + ppServerEnd = pGroup->active_servers + pGroup->active_count; + for (ppServer=pGroup->active_servers; ppServerid, pDestServer->id) == 0) + { + continue; + } + + return *ppServer; + } + + return NULL; +} + +static int tracker_mem_realloc_store_servers(FDFSGroupInfo *pGroup, \ + const int inc_count, const bool bNeedSleep) +{ + int result; + FDFSStorageDetail **old_servers; + FDFSStorageDetail **old_sorted_servers; + FDFSStorageDetail **old_active_servers; + int **old_last_sync_timestamps; + FDFSStorageDetail **new_servers; + FDFSStorageDetail **new_sorted_servers; + FDFSStorageDetail **new_active_servers; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; +#ifdef WITH_HTTPD + FDFSStorageDetail **old_http_servers; + FDFSStorageDetail **new_http_servers; +#endif + int **new_last_sync_timestamps; + int old_size; + int new_size; + int err_no; + int i; + + new_size = pGroup->alloc_size + inc_count + TRACKER_MEM_ALLOC_ONCE; + new_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * new_size); + if (new_servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(FDFSStorageDetail *) * new_size); + return errno != 0 ? errno : ENOMEM; + } + memset(new_servers, 0, sizeof(FDFSStorageDetail *) * new_size); + + ppServerEnd = new_servers + new_size; + for (ppServer=new_servers+pGroup->count; ppServerall_servers, \ + sizeof(FDFSStorageDetail *) * pGroup->count); + + new_sorted_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * new_size); + if (new_sorted_servers == NULL) + { + free(new_servers); + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(FDFSStorageDetail *) * new_size); + return errno != 0 ? errno : ENOMEM; + } + + new_active_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * new_size); + if (new_active_servers == NULL) + { + free(new_servers); + free(new_sorted_servers); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(FDFSStorageDetail *) * new_size); + return errno != 0 ? errno : ENOMEM; + } + +#ifdef WITH_HTTPD + if (g_http_check_interval > 0) + { + new_http_servers = (FDFSStorageDetail **) \ + malloc(sizeof(FDFSStorageDetail *) * new_size); + if (new_http_servers == NULL) + { + free(new_servers); + free(new_sorted_servers); + free(new_active_servers); + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSStorageDetail *) * new_size); + return errno != 0 ? errno : ENOMEM; + } + + memset(new_http_servers,0,sizeof(FDFSStorageDetail *)*new_size); + + memcpy(new_http_servers, pGroup->http_servers, \ + sizeof(FDFSStorageDetail *) * pGroup->count); + + } + else + { + new_http_servers = NULL; + } +#endif + + memset(new_sorted_servers, 0, sizeof(FDFSStorageDetail *) * new_size); + memset(new_active_servers, 0, sizeof(FDFSStorageDetail *) * new_size); + if (pGroup->store_path_count > 0) + { + for (i=pGroup->count; istore_path_count); + if (result != 0) + { + free(new_servers); + free(new_sorted_servers); + + return result; + } + } + } + + memcpy(new_sorted_servers, pGroup->sorted_servers, \ + sizeof(FDFSStorageDetail *) * pGroup->count); + + memcpy(new_active_servers, pGroup->active_servers, \ + sizeof(FDFSStorageDetail *) * pGroup->count); + + new_last_sync_timestamps = tracker_malloc_last_sync_timestamps( \ + new_size, &err_no); + if (new_last_sync_timestamps == NULL) + { + free(new_servers); + free(new_sorted_servers); + free(new_active_servers); + + return err_no; + } + for (i=0; ialloc_size; i++) + { + memcpy(new_last_sync_timestamps[i], \ + pGroup->last_sync_timestamps[i], \ + (int)sizeof(int) * pGroup->alloc_size); + } + + old_size = pGroup->alloc_size; + old_servers = pGroup->all_servers; + old_sorted_servers = pGroup->sorted_servers; + old_active_servers = pGroup->active_servers; + old_last_sync_timestamps = pGroup->last_sync_timestamps; + + pGroup->alloc_size = new_size; + pGroup->all_servers = new_servers; + pGroup->sorted_servers = new_sorted_servers; + pGroup->active_servers = new_active_servers; + pGroup->last_sync_timestamps = new_last_sync_timestamps; + + tracker_mem_find_store_server(pGroup); + if (g_if_leader_self && g_if_use_trunk_file) + { + tracker_mem_find_trunk_server(pGroup, true); + } + +#ifdef WITH_HTTPD + if (g_http_check_interval <= 0) + { + old_http_servers = NULL; + pGroup->http_servers = pGroup->active_servers; + } + else + { + old_http_servers = pGroup->http_servers; + pGroup->http_servers = new_http_servers; + g_http_servers_dirty = true; + } +#endif + + if (bNeedSleep) + { + sleep(1); + } + + free(old_servers); + + free(old_sorted_servers); + free(old_active_servers); + +#ifdef WITH_HTTPD + if (old_http_servers != NULL) + { + free(old_http_servers); + } +#endif + + tracker_free_last_sync_timestamps(old_last_sync_timestamps, \ + old_size); + + return 0; +} + +static int tracker_mem_cmp_by_group_name(const void *p1, const void *p2) +{ + return strcmp((*((FDFSGroupInfo **)p1))->group_name, + (*((FDFSGroupInfo **)p2))->group_name); +} + +static int tracker_mem_cmp_by_storage_id(const void *p1, const void *p2) +{ + return strcmp((*((FDFSStorageDetail **)p1))->id, + (*((FDFSStorageDetail **)p2))->id); +} + +static void tracker_mem_insert_into_sorted_servers( \ + FDFSStorageDetail *pTargetServer, \ + FDFSStorageDetail **sorted_servers, const int count) +{ + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppEnd; + + ppEnd = sorted_servers + count; + for (ppServer=ppEnd; ppServer>sorted_servers; ppServer--) + { + if (strcmp(pTargetServer->id, (*(ppServer-1))->id) > 0) + { + break; + } + else + { + *ppServer = *(ppServer-1); + } + } + + *ppServer = pTargetServer; +} + +static void tracker_mem_insert_into_sorted_groups(FDFSGroups *pGroups, \ + FDFSGroupInfo *pTargetGroup) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + + ppEnd = pGroups->sorted_groups + pGroups->count; + for (ppGroup=ppEnd; ppGroup > pGroups->sorted_groups; ppGroup--) + { + if (strcmp(pTargetGroup->group_name, \ + (*(ppGroup-1))->group_name) > 0) + { + *ppGroup = pTargetGroup; + return; + } + else + { + *ppGroup = *(ppGroup-1); + } + } + + *ppGroup = pTargetGroup; +} + +FDFSGroupInfo *tracker_mem_get_group_ex(FDFSGroups *pGroups, \ + const char *group_name) +{ + FDFSGroupInfo target_groups; + FDFSGroupInfo *pTargetGroups; + FDFSGroupInfo **ppGroup; + + memset(&target_groups, 0, sizeof(target_groups)); + strcpy(target_groups.group_name, group_name); + pTargetGroups = &target_groups; + ppGroup = (FDFSGroupInfo **)bsearch(&pTargetGroups, \ + pGroups->sorted_groups, \ + pGroups->count, sizeof(FDFSGroupInfo *), \ + tracker_mem_cmp_by_group_name); + + if (ppGroup != NULL) + { + return *ppGroup; + } + else + { + return NULL; + } +} + +static int tracker_mem_add_group_ex(FDFSGroups *pGroups, \ + TrackerClientInfo *pClientInfo, const char *group_name, \ + const bool bNeedSleep, bool *bInserted) +{ + FDFSGroupInfo *pGroup; + int result; + + if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + do + { + result = 0; + *bInserted = false; + pGroup = tracker_mem_get_group_ex(pGroups, group_name); + if (pGroup != NULL) + { + break; + } + + if (pGroups->count >= pGroups->alloc_size) + { + result = tracker_mem_realloc_groups(pGroups, bNeedSleep); + if (result != 0) + { + break; + } + } + + pGroup = *(pGroups->groups + pGroups->count); + result = tracker_mem_init_group(pGroup); + if (result != 0) + { + break; + } + + strcpy(pGroup->group_name, group_name); + tracker_mem_insert_into_sorted_groups(pGroups, pGroup); + pGroups->count++; + + if ((pGroups->store_lookup == \ + FDFS_STORE_LOOKUP_SPEC_GROUP) && \ + (pGroups->pStoreGroup == NULL) && \ + (strcmp(pGroups->store_group, \ + pGroup->group_name) == 0)) + { + pGroups->pStoreGroup = pGroup; + } + + *bInserted = true; + } while (0); + + if (pthread_mutex_unlock(&mem_thread_lock) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail", \ + __LINE__); + } + + if (result != 0) + { + return result; + } + + pClientInfo->pGroup = pGroup; + return 0; +} + +static FDFSStorageDetail *tracker_mem_get_active_storage_by_id( \ + FDFSGroupInfo *pGroup, const char *id) +{ + FDFSStorageDetail target_storage; + FDFSStorageDetail *pTargetStorage; + FDFSStorageDetail **ppStorageServer; + + if (id == NULL) + { + return NULL; + } + + memset(&target_storage, 0, sizeof(target_storage)); + strcpy(target_storage.id, id); + pTargetStorage = &target_storage; + ppStorageServer = (FDFSStorageDetail **)bsearch(&pTargetStorage, \ + pGroup->active_servers, \ + pGroup->active_count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer != NULL) + { + return *ppStorageServer; + } + else + { + return NULL; + } +} + +static FDFSStorageDetail *tracker_mem_get_active_storage_by_ip( \ + FDFSGroupInfo *pGroup, const char *ip_addr) +{ + FDFSStorageIdInfo *pStorageId; + + if (!g_use_storage_id) + { + return tracker_mem_get_active_storage_by_id(pGroup, ip_addr); + } + + pStorageId = fdfs_get_storage_id_by_ip(pGroup->group_name, ip_addr); + if (pStorageId == NULL) + { + return NULL; + } + return tracker_mem_get_active_storage_by_id(pGroup, pStorageId->id); +} + +#ifdef WITH_HTTPD +static FDFSStorageDetail *tracker_mem_get_active_http_server_by_ip( \ + FDFSGroupInfo *pGroup, const char *ip_addr) +{ + FDFSStorageDetail target_storage; + FDFSStorageDetail *pTargetStorage; + FDFSStorageDetail **ppStorageServer; + + memset(&target_storage, 0, sizeof(target_storage)); + if (!g_use_storage_id) + { + strcpy(target_storage.id, ip_addr); + } + else + { + FDFSStorageIdInfo *pStorageId; + pStorageId = fdfs_get_storage_id_by_ip( \ + pGroup->group_name, ip_addr); + if (pStorageId == NULL) + { + return NULL; + } + strcpy(target_storage.id, pStorageId->id); + } + pTargetStorage = &target_storage; + ppStorageServer = (FDFSStorageDetail **)bsearch(&pTargetStorage, \ + pGroup->http_servers, \ + pGroup->http_server_count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer != NULL) + { + return *ppStorageServer; + } + else + { + return NULL; + } +} + +static FDFSStorageDetail *tracker_mem_get_active_http_server_by_id( \ + FDFSGroupInfo *pGroup, const char *storage_id) +{ + FDFSStorageDetail target_storage; + FDFSStorageDetail *pTargetStorage; + FDFSStorageDetail **ppStorageServer; + + memset(&target_storage, 0, sizeof(target_storage)); + strcpy(target_storage.id, storage_id); + pTargetStorage = &target_storage; + ppStorageServer = (FDFSStorageDetail **)bsearch(&pTargetStorage, \ + pGroup->http_servers, \ + pGroup->http_server_count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer != NULL) + { + return *ppStorageServer; + } + else + { + return NULL; + } +} +#endif + +FDFSStorageDetail *tracker_mem_get_storage_by_ip(FDFSGroupInfo *pGroup, \ + const char *ip_addr) +{ + const char *storage_id; + + if (g_use_storage_id) + { + FDFSStorageIdInfo *pStorageIdInfo; + pStorageIdInfo = fdfs_get_storage_id_by_ip( \ + pGroup->group_name, ip_addr); + if (pStorageIdInfo == NULL) + { + return NULL; + } + storage_id = pStorageIdInfo->id; + } + else + { + storage_id = ip_addr; + } + + return tracker_mem_get_storage(pGroup, storage_id); +} + +FDFSStorageDetail *tracker_mem_get_storage(FDFSGroupInfo *pGroup, \ + const char *id) +{ + FDFSStorageDetail target_storage; + FDFSStorageDetail *pTargetStorage; + FDFSStorageDetail **ppStorageServer; + + memset(&target_storage, 0, sizeof(target_storage)); + strcpy(target_storage.id, id); + pTargetStorage = &target_storage; + ppStorageServer = (FDFSStorageDetail **)bsearch(&pTargetStorage, \ + pGroup->sorted_servers, \ + pGroup->count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer != NULL) + { + return *ppStorageServer; + } + else + { + return NULL; + } +} + +static void tracker_mem_clear_storage_fields(FDFSStorageDetail *pStorageServer) +{ + if (pStorageServer->path_total_mbs != NULL) + { + memset(pStorageServer->path_total_mbs, 0, sizeof(int64_t) \ + * pStorageServer->store_path_count); + } + + if (pStorageServer->path_free_mbs != NULL) + { + memset(pStorageServer->path_free_mbs, 0, sizeof(int64_t) \ + * pStorageServer->store_path_count); + } + + pStorageServer->psync_src_server = NULL; + pStorageServer->sync_until_timestamp = 0; + pStorageServer->total_mb = 0; + pStorageServer->free_mb = 0; + pStorageServer->changelog_offset = 0; + pStorageServer->store_path_count = 0; + pStorageServer->subdir_count_per_path = 0; + pStorageServer->upload_priority = 0; + pStorageServer->current_write_path = 0; + + memset(&(pStorageServer->stat), 0, sizeof(FDFSStorageStat)); +} + +int tracker_mem_delete_storage(FDFSGroupInfo *pGroup, const char *id) +{ + FDFSStorageDetail *pStorageServer; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppEnd; + + pStorageServer = tracker_mem_get_storage(pGroup, id); + if (pStorageServer == NULL || pStorageServer->status == \ + FDFS_STORAGE_STATUS_IP_CHANGED) + { + return ENOENT; + } + + if (pStorageServer->status == FDFS_STORAGE_STATUS_ONLINE || \ + pStorageServer->status == FDFS_STORAGE_STATUS_ACTIVE || \ + pStorageServer->status == FDFS_STORAGE_STATUS_RECOVERY) + { + return EBUSY; + } + + if (pStorageServer->status == FDFS_STORAGE_STATUS_DELETED) + { + return EALREADY; + } + + ppEnd = pGroup->all_servers + pGroup->count; + for (ppServer=pGroup->all_servers; ppServerpsync_src_server != NULL && \ + strcmp((*ppServer)->psync_src_server->id, id) == 0) + { + (*ppServer)->psync_src_server = NULL; + } + } + + tracker_mem_clear_storage_fields(pStorageServer); + + pStorageServer->status = FDFS_STORAGE_STATUS_DELETED; + pGroup->chg_count++; + + tracker_write_to_changelog(pGroup, pStorageServer, NULL); + return 0; +} + +int tracker_mem_storage_ip_changed(FDFSGroupInfo *pGroup, \ + const char *old_storage_ip, const char *new_storage_ip) +{ + FDFSStorageDetail *pOldStorageServer; + FDFSStorageDetail *pNewStorageServer; + int result; + bool bInserted; + + if (g_use_storage_id) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, do NOT support ip changed adjust " \ + "because cluster use server ID instead of " \ + "IP address", __LINE__, new_storage_ip); + return EOPNOTSUPP; + } + + pOldStorageServer = tracker_mem_get_storage(pGroup, old_storage_ip); + if (pOldStorageServer == NULL || pOldStorageServer->status == \ + FDFS_STORAGE_STATUS_DELETED) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, old storage server: %s not exists", \ + __LINE__, new_storage_ip, old_storage_ip); + return ENOENT; + } + + if (pOldStorageServer->status == FDFS_STORAGE_STATUS_ONLINE || \ + pOldStorageServer->status == FDFS_STORAGE_STATUS_ACTIVE || \ + pOldStorageServer->status == FDFS_STORAGE_STATUS_RECOVERY) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, old storage server: %s is online", \ + __LINE__, new_storage_ip, old_storage_ip); + return EBUSY; + } + + if (pOldStorageServer->status == FDFS_STORAGE_STATUS_IP_CHANGED) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, old storage server: %s " \ + "'s ip address already changed", \ + __LINE__, new_storage_ip, old_storage_ip); + return EALREADY; + } + + pNewStorageServer = tracker_mem_get_storage(pGroup, new_storage_ip); + if (!(pNewStorageServer == NULL || pNewStorageServer->status == \ + FDFS_STORAGE_STATUS_DELETED)) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, new storage server: %s already exists",\ + __LINE__, new_storage_ip, new_storage_ip); + return EEXIST; + } + + result = _tracker_mem_add_storage(pGroup, &pNewStorageServer, \ + new_storage_ip, new_storage_ip, true, true, &bInserted); + if (result != 0) + { + return result; + } + + if (!bInserted) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, new storage server: %s already exists",\ + __LINE__, new_storage_ip, new_storage_ip); + return EEXIST; + } + + pthread_mutex_lock(&mem_thread_lock); + + //exchange old and new storage server + snprintf(pOldStorageServer->id, sizeof(pOldStorageServer->id), \ + "%s", new_storage_ip); + snprintf(pOldStorageServer->ip_addr, \ + sizeof(pOldStorageServer->ip_addr), "%s", new_storage_ip); + + snprintf(pNewStorageServer->id, sizeof(pNewStorageServer->id), \ + "%s", old_storage_ip); + snprintf(pNewStorageServer->ip_addr, \ + sizeof(pNewStorageServer->ip_addr), "%s", old_storage_ip); + pNewStorageServer->status = FDFS_STORAGE_STATUS_IP_CHANGED; + + pGroup->chg_count++; + + //need re-sort + qsort(pGroup->sorted_servers, pGroup->count, \ + sizeof(FDFSStorageDetail *), tracker_mem_cmp_by_storage_id); + + pthread_mutex_unlock(&mem_thread_lock); + + tracker_write_to_changelog(pGroup, pNewStorageServer, new_storage_ip); + + return tracker_save_sys_files(); +} + +static int tracker_mem_add_storage(TrackerClientInfo *pClientInfo, \ + const char *id, const char *ip_addr, \ + const bool bNeedSleep, const bool bNeedLock, bool *bInserted) +{ + int result; + FDFSStorageDetail *pStorageServer; + + pStorageServer = NULL; + result = _tracker_mem_add_storage(pClientInfo->pGroup, \ + &pStorageServer, id, ip_addr, bNeedSleep, \ + bNeedLock, bInserted); + if (result == 0) + { + pClientInfo->pStorage = pStorageServer; + } + + return result; +} + +static int _tracker_mem_add_storage(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail **ppStorageServer, const char *id, \ + const char *ip_addr, const bool bNeedSleep, \ + const bool bNeedLock, bool *bInserted) +{ + int result; + const char *storage_id; + + if (*ip_addr == '\0') + { + logError("file: "__FILE__", line: %d, " \ + "ip address is empty!", __LINE__); + return EINVAL; + } + + if (id != NULL) + { + if (g_use_storage_id) + { + result = fdfs_check_storage_id( \ + pGroup->group_name, id); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "check storage id fail, " \ + "group_name: %s, id: %s, " \ + "storage ip: %s, errno: %d, " \ + "error info: %s", __LINE__, \ + pGroup->group_name, id, ip_addr, \ + result, STRERROR(result)); + return result; + } + } + + storage_id = id; + } + else if (g_use_storage_id) + { + FDFSStorageIdInfo *pStorageIdInfo; + pStorageIdInfo = fdfs_get_storage_id_by_ip( \ + pGroup->group_name, ip_addr); + if (pStorageIdInfo == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "get storage id info fail, " \ + "group_name: %s, storage ip: %s", \ + __LINE__, pGroup->group_name, ip_addr); + return ENOENT; + } + storage_id = pStorageIdInfo->id; + } + else + { + storage_id = ip_addr; + } + + if (bNeedLock && (result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + do + { + result = 0; + *bInserted = false; + *ppStorageServer = tracker_mem_get_storage(pGroup, storage_id); + if (*ppStorageServer != NULL) + { + if (g_use_storage_id) + { + memcpy ((*ppStorageServer)->ip_addr, ip_addr, \ + IP_ADDRESS_SIZE); + } + + if ((*ppStorageServer)->status==FDFS_STORAGE_STATUS_DELETED \ + || (*ppStorageServer)->status==FDFS_STORAGE_STATUS_IP_CHANGED) + { + (*ppStorageServer)->status = FDFS_STORAGE_STATUS_INIT; + } + + break; + } + + if (pGroup->count >= pGroup->alloc_size) + { + result = tracker_mem_realloc_store_servers( \ + pGroup, 1, bNeedSleep); + if (result != 0) + { + break; + } + } + + *ppStorageServer = *(pGroup->all_servers + pGroup->count); + snprintf((*ppStorageServer)->id, FDFS_STORAGE_ID_MAX_SIZE, \ + "%s", storage_id); + memcpy((*ppStorageServer)->ip_addr, ip_addr, IP_ADDRESS_SIZE); + + tracker_mem_insert_into_sorted_servers(*ppStorageServer, \ + pGroup->sorted_servers, pGroup->count); + pGroup->count++; + pGroup->chg_count++; + + *bInserted = true; + } while (0); + + if (bNeedLock && pthread_mutex_unlock(&mem_thread_lock) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail", \ + __LINE__); + } + + return result; +} + +int tracker_mem_get_status(ConnectionInfo *pTrackerServer, \ + TrackerRunningStatus *pStatus) +{ + char in_buff[1 + 2 * FDFS_PROTO_PKG_LEN_SIZE]; + TrackerHeader header; + char *pInBuff; + ConnectionInfo *conn; + int64_t in_bytes; + int result; + + pTrackerServer->sock = -1; + if ((conn=tracker_connect_server(pTrackerServer, &result)) == NULL) + { + return result; + } + + do + { + memset(&header, 0, sizeof(header)); + header.cmd = TRACKER_PROTO_CMD_TRACKER_GET_STATUS; + if ((result=tcpsenddata_nb(conn->sock, &header, \ + sizeof(header), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + + result = (result == ENOENT ? EACCES : result); + break; + } + + pInBuff = in_buff; + result = fdfs_recv_response(conn, &pInBuff, \ + sizeof(in_buff), &in_bytes); + if (result != 0) + { + break; + } + + if (in_bytes != sizeof(in_buff)) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length: %d.", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + in_bytes, (int)sizeof(in_buff)); + result = EINVAL; + break; + } + + pStatus->if_leader = *in_buff; + pStatus->running_time = buff2long(in_buff + 1); + pStatus->restart_interval = buff2long(in_buff + 1 + \ + FDFS_PROTO_PKG_LEN_SIZE); + + } while (0); + + tracker_disconnect_server_ex(conn, result != 0); + + return result; +} + +void tracker_calc_running_times(TrackerRunningStatus *pStatus) +{ + pStatus->running_time = g_current_time - g_up_time; + + if (g_tracker_last_status.last_check_time == 0) + { + pStatus->restart_interval = 0; + } + else + { + pStatus->restart_interval = g_up_time - \ + g_tracker_last_status.last_check_time; + } + +#define FDFS_TRIM_TIME(t, i) (t / i) * i + + pStatus->running_time = FDFS_TRIM_TIME(pStatus->running_time, \ + TRACKER_SYNC_STATUS_FILE_INTERVAL); + pStatus->restart_interval = FDFS_TRIM_TIME(pStatus->restart_interval, \ + TRACKER_SYNC_STATUS_FILE_INTERVAL); +} + +static int tracker_mem_get_sys_file_piece(ConnectionInfo *pTrackerServer, \ + const int file_index, int fd, int64_t *offset, int64_t *file_size) +{ + char out_buff[sizeof(TrackerHeader) + 1 + FDFS_PROTO_PKG_LEN_SIZE]; + char in_buff[TRACKER_MAX_PACKAGE_SIZE]; + TrackerHeader *pHeader; + char *p; + char *pInBuff; + char *pContent; + int64_t in_bytes; + int64_t write_bytes; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + pHeader->cmd = TRACKER_PROTO_CMD_TRACKER_GET_ONE_SYS_FILE; + long2buff(1 + FDFS_PROTO_PKG_LEN_SIZE, pHeader->pkg_len); + + p = out_buff + sizeof(TrackerHeader); + *p++ = file_index; + long2buff(*offset, p); + if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + + return (result == ENOENT ? EACCES : result); + } + + pInBuff = in_buff; + result = fdfs_recv_response(pTrackerServer, &pInBuff, \ + sizeof(in_buff), &in_bytes); + if (result != 0) + { + return result; + } + + if (in_bytes < FDFS_PROTO_PKG_LEN_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length >= %d.", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + in_bytes, FDFS_PROTO_PKG_LEN_SIZE); + return EINVAL; + } + + *file_size = buff2long(in_buff); + write_bytes = in_bytes - FDFS_PROTO_PKG_LEN_SIZE; + + if (*file_size < 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, file size: "INT64_PRINTF_FORMAT\ + " < 0", __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, *file_size); + return EINVAL; + } + + if (*file_size > 0 && write_bytes == 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, file size: "INT64_PRINTF_FORMAT\ + " > 0, but file content is empty", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + *file_size); + return EINVAL; + } + + pContent = pInBuff + FDFS_PROTO_PKG_LEN_SIZE; + if (write_bytes > 0 && write(fd, pContent, write_bytes) != write_bytes) + { + logError("file: "__FILE__", line: %d, " \ + "write to file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, g_tracker_sys_filenames[file_index], \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EIO; + } + + *offset += write_bytes; + return 0; +} + +static int tracker_mem_get_one_sys_file(ConnectionInfo *pTrackerServer, \ + const int file_index) +{ + char full_filename[MAX_PATH_SIZE]; + int fd; + int result; + int64_t offset; + int64_t file_size; + + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + g_fdfs_base_path, g_tracker_sys_filenames[file_index]); + fd = open(full_filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) + { + logError("file: "__FILE__", line: %d, " \ + "open file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + return errno != 0 ? errno : EACCES; + } + + TRACKER_FCHOWN(fd, full_filename, geteuid(), getegid()) + + offset = 0; + file_size = 0; + while (1) + { + result = tracker_mem_get_sys_file_piece(pTrackerServer, \ + file_index, fd, &offset, &file_size); + if (result != 0) + { + break; + } + + if (offset >= file_size) + { + break; + } + } + + close(fd); + return result; +} + +static int tracker_mem_get_sys_files(ConnectionInfo *pTrackerServer) +{ + ConnectionInfo *conn; + int result; + int index; + + pTrackerServer->sock = -1; + if ((conn=tracker_connect_server(pTrackerServer, &result)) == NULL) + { + return result; + } + + if ((result=tracker_get_sys_files_start(conn)) != 0) + { + tracker_disconnect_server_ex(conn, true); + return result; + } + + for (index=0; indexrunning_time - pStatus2->running_time; + if (sub != 0) + { + return sub; + } + + return pStatus2->restart_interval - pStatus1->restart_interval; +} + +static int tracker_mem_first_add_tracker_servers(FDFSStorageJoinBody *pJoinBody) +{ + ConnectionInfo *pLocalTracker; + ConnectionInfo *pLocalEnd; + ConnectionInfo *servers; + int tracker_count; + int bytes; + + tracker_count = pJoinBody->tracker_count; + bytes = sizeof(ConnectionInfo) * tracker_count; + servers = (ConnectionInfo *)malloc(bytes); + if (servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + memcpy(servers, pJoinBody->tracker_servers, bytes); + + pLocalEnd = servers + tracker_count; + for (pLocalTracker=servers; pLocalTrackersock = -1; + } + + g_tracker_servers.servers = servers; + g_tracker_servers.server_count = tracker_count; + return 0; +} + +static int tracker_mem_check_add_tracker_servers(FDFSStorageJoinBody *pJoinBody) +{ + ConnectionInfo *pJoinTracker; + ConnectionInfo *pJoinEnd; + ConnectionInfo *pLocalTracker; + ConnectionInfo *pLocalEnd; + ConnectionInfo *pNewServer; + ConnectionInfo *new_servers; + int add_count; + int bytes; + + add_count = 0; + pLocalEnd = g_tracker_servers.servers + g_tracker_servers.server_count; + pJoinEnd = pJoinBody->tracker_servers + pJoinBody->tracker_count; + for (pJoinTracker=pJoinBody->tracker_servers; \ + pJoinTrackerport == pLocalTracker->port && \ + strcmp(pJoinTracker->ip_addr, \ + pLocalTracker->ip_addr) == 0) + { + break; + } + } + + if (pLocalTracker == pLocalEnd) + { + add_count++; + } + } + + if (add_count == 0) + { + return 0; + } + + if (g_last_tracker_servers != NULL) + { + logError("file: "__FILE__", line: %d, " \ + "last tracker servers does not freed, " \ + "should try again!", __LINE__); + return EAGAIN; + } + + if (g_tracker_servers.server_count + add_count > FDFS_MAX_TRACKERS) + { + logError("file: "__FILE__", line: %d, " \ + "too many tracker servers: %d", \ + __LINE__, g_tracker_servers.server_count + add_count); + return ENOSPC; + } + + logInfo("file: "__FILE__", line: %d, " \ + "add %d tracker servers", \ + __LINE__, add_count); + + bytes = sizeof(ConnectionInfo) * (g_tracker_servers.server_count \ + + add_count); + new_servers = (ConnectionInfo *)malloc(bytes); + if (new_servers == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, " \ + "errno: %d, error info: %s", \ + __LINE__, bytes, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + memcpy(new_servers, g_tracker_servers.servers, sizeof(ConnectionInfo)* \ + g_tracker_servers.server_count); + pNewServer = new_servers + g_tracker_servers.server_count; + for (pJoinTracker=pJoinBody->tracker_servers; \ + pJoinTrackerport == pLocalTracker->port && \ + strcmp(pJoinTracker->ip_addr, \ + pLocalTracker->ip_addr) == 0) + { + break; + } + } + + if (pLocalTracker == pLocalEnd) + { + memcpy(pNewServer, pJoinTracker, \ + sizeof(ConnectionInfo)); + pNewServer->sock = -1; + pNewServer++; + } + } + + g_last_tracker_servers = g_tracker_servers.servers; + g_tracker_servers.servers = new_servers; + g_tracker_servers.server_count += add_count; + + return 0; +} + +static int tracker_mem_get_tracker_server(FDFSStorageJoinBody *pJoinBody, \ + TrackerRunningStatus *pTrackerStatus) +{ + ConnectionInfo *pTrackerServer; + ConnectionInfo *pTrackerEnd; + TrackerRunningStatus *pStatus; + TrackerRunningStatus trackerStatus[FDFS_MAX_TRACKERS]; + int count; + int result; + int r; + int i; + + memset(pTrackerStatus, 0, sizeof(TrackerRunningStatus)); + pStatus = trackerStatus; + result = 0; + pTrackerEnd = pJoinBody->tracker_servers + pJoinBody->tracker_count; + for (pTrackerServer=pJoinBody->tracker_servers; \ + pTrackerServerport == g_server_port && \ + is_local_host_ip(pTrackerServer->ip_addr)) + { + continue; + } + + pStatus->pTrackerServer = pTrackerServer; + r = tracker_mem_get_status(pTrackerServer, pStatus); + if (r == 0) + { + pStatus++; + } + else if (r != ENOENT) + { + result = r; + } + } + + count = pStatus - trackerStatus; + if (count == 0) + { + return result == 0 ? ENOENT : result; + } + + if (count == 1) + { + memcpy(pTrackerStatus, trackerStatus, \ + sizeof(TrackerRunningStatus)); + return 0; + } + + qsort(trackerStatus, count, sizeof(TrackerRunningStatus), \ + tracker_mem_cmp_tracker_running_status); + + for (i=0; iip_addr, \ + trackerStatus[i].pTrackerServer->port, \ + trackerStatus[i].running_time, \ + trackerStatus[i].restart_interval); + } + + //copy the last + memcpy(pTrackerStatus, trackerStatus + (count - 1), \ + sizeof(TrackerRunningStatus)); + return 0; +} + +static int tracker_mem_get_sys_files_from_others(FDFSStorageJoinBody *pJoinBody, + TrackerRunningStatus *pRunningStatus) +{ + int result; + TrackerRunningStatus trackerStatus; + ConnectionInfo *pTrackerServer; + FDFSGroups newGroups; + FDFSGroups tempGroups; + + if (pJoinBody->tracker_count == 0) + { + return 0; + } + + result = tracker_mem_get_tracker_server(pJoinBody, &trackerStatus); + if (result != 0) + { + return result == ENOENT ? 0 : result; + } + + if (pRunningStatus != NULL) + { + if (tracker_mem_cmp_tracker_running_status(pRunningStatus, \ + &trackerStatus) >= 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "%s:%d running time: %d, restart interval: %d, "\ + "my running time: %d, restart interval: %d, " \ + "do not need sync system files", __LINE__, \ + trackerStatus.pTrackerServer->ip_addr, \ + trackerStatus.pTrackerServer->port, \ + trackerStatus.running_time, \ + trackerStatus.restart_interval, \ + pRunningStatus->running_time, \ + pRunningStatus->restart_interval); + + return 0; + } + } + + pTrackerServer = trackerStatus.pTrackerServer; + result = tracker_mem_get_sys_files(pTrackerServer); + if (result != 0) + { + return result; + } + + logInfo("file: "__FILE__", line: %d, " \ + "sys files loaded from tracker server %s:%d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port); + + memset(&newGroups, 0, sizeof(newGroups)); + newGroups.store_lookup = g_groups.store_lookup; + newGroups.store_server = g_groups.store_server; + newGroups.download_server = g_groups.download_server; + newGroups.store_path = g_groups.store_path; + strcpy(newGroups.store_group, g_groups.store_group); + if ((result=tracker_mem_init_groups(&newGroups)) != 0) + { + tracker_mem_destroy_groups(&newGroups, false); + return result; + } + + memcpy(&tempGroups, &g_groups, sizeof(FDFSGroups)); + memcpy(&g_groups, &newGroups, sizeof(FDFSGroups)); + + usleep(100000); + + tracker_mem_destroy_groups(&tempGroups, false); + tracker_write_status_to_file(NULL); + + if (changelog_fd >= 0) + { + close(changelog_fd); + changelog_fd = -1; + } + + return tracker_open_changlog_file(); +} + +int tracker_mem_add_group_and_storage(TrackerClientInfo *pClientInfo, \ + const char *ip_addr, FDFSStorageJoinBody *pJoinBody, \ + const bool bNeedSleep) +{ + int result; + bool bStorageInserted; + bool bGroupInserted; + FDFSStorageDetail *pStorageServer; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppEnd; + FDFSStorageIdInfo *pStorageIdInfo; + const char *storage_id; + + tracker_mem_file_lock(); + + if (need_get_sys_files) + { + if (g_tracker_last_status.last_check_time > 0 && \ + g_up_time - g_tracker_last_status.last_check_time > \ + 2 * TRACKER_SYNC_STATUS_FILE_INTERVAL) + { /* stop time exceeds 2 * interval */ + TrackerRunningStatus runningStatus; + + tracker_calc_running_times(&runningStatus); + result = tracker_mem_get_sys_files_from_others(\ + pJoinBody, &runningStatus); + if (result != 0) + { + tracker_mem_file_unlock(); + return EAGAIN; + } + + get_sys_files_done = true; + } + + need_get_sys_files = false; + } + + if ((!get_sys_files_done) && (g_groups.count == 0)) + { + if (g_groups.count == 0) + { + if ((result=tracker_mem_get_sys_files_from_others( \ + pJoinBody, NULL)) != 0) + { + tracker_mem_file_unlock(); + return EAGAIN; + } + + get_sys_files_done = true; + } + } + + if (g_tracker_servers.servers == NULL) + { + result = tracker_mem_first_add_tracker_servers(pJoinBody); + if (result != 0) + { + tracker_mem_file_unlock(); + return result; + } + } + else + { + result = tracker_mem_check_add_tracker_servers(pJoinBody); + if (result != 0) + { + tracker_mem_file_unlock(); + return result; + } + } + + tracker_mem_file_unlock(); + + if ((result=tracker_mem_add_group_ex(&g_groups, pClientInfo, \ + pJoinBody->group_name, bNeedSleep, &bGroupInserted)) != 0) + { + return result; + } + + if (bGroupInserted) + { + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + + if (g_use_storage_id) + { + pStorageIdInfo = fdfs_get_storage_id_by_ip( \ + pClientInfo->pGroup->group_name, ip_addr); + if (pStorageIdInfo == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "get storage id info fail, group_name: %s, " \ + "storage ip: %s", __LINE__, \ + pClientInfo->pGroup->group_name, ip_addr); + return ENOENT; + } + storage_id = pStorageIdInfo->id; + } + else + { + pStorageIdInfo = NULL; + storage_id = ip_addr; + } + + if (pClientInfo->pGroup->storage_port == 0) + { + pClientInfo->pGroup->storage_port = pJoinBody->storage_port; + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + if (pClientInfo->pGroup->storage_port != \ + pJoinBody->storage_port) + { + ppEnd = pClientInfo->pGroup->all_servers + \ + pClientInfo->pGroup->count; + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServerid, storage_id) == 0) + { + (*ppServer)->storage_port = \ + pJoinBody->storage_port; + break; + } + } + + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServerstorage_port != \ + pJoinBody->storage_port) + { + break; + } + } + if (ppServer == ppEnd) //all servers are same, adjust + { + pClientInfo->pGroup->storage_port = \ + pJoinBody->storage_port; + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, port %d is not same " \ + "in the group \"%s\", group port is %d", \ + __LINE__, ip_addr, pJoinBody->storage_port, \ + pJoinBody->group_name, \ + pClientInfo->pGroup->storage_port); + return EINVAL; + } + } + } + + if (pClientInfo->pGroup->storage_http_port == 0) + { + pClientInfo->pGroup->storage_http_port = \ + pJoinBody->storage_http_port; + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + if (pClientInfo->pGroup->storage_http_port != \ + pJoinBody->storage_http_port) + { + ppEnd = pClientInfo->pGroup->all_servers + \ + pClientInfo->pGroup->count; + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServerid, storage_id) == 0) + { + (*ppServer)->storage_http_port = \ + pJoinBody->storage_http_port; + break; + } + } + + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServerstorage_http_port != \ + pJoinBody->storage_http_port) + { + break; + } + } + if (ppServer == ppEnd) //all servers are same, adjust + { + pClientInfo->pGroup->storage_http_port = \ + pJoinBody->storage_http_port; + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, http port %d is not same " \ + "in the group \"%s\", group http port is %d", \ + __LINE__, ip_addr, \ + pJoinBody->storage_http_port, \ + pJoinBody->group_name, \ + pClientInfo->pGroup->storage_http_port); +#ifdef WITH_HTTPD + return EINVAL; +#endif + } + } + } + + if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + pStorageServer = tracker_mem_get_storage(pClientInfo->pGroup, storage_id); + if (pthread_mutex_unlock(&mem_thread_lock) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail", \ + __LINE__); + } + + if (pStorageServer == NULL) + { + if (!pJoinBody->init_flag) + { + if (pJoinBody->status < 0 || \ + pJoinBody->status == FDFS_STORAGE_STATUS_DELETED || \ + pJoinBody->status == FDFS_STORAGE_STATUS_IP_CHANGED || \ + pJoinBody->status == FDFS_STORAGE_STATUS_NONE) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s:%d, invalid storage " \ + "status %d, in the group \"%s\"", \ + __LINE__, ip_addr, \ + pJoinBody->storage_port, \ + pJoinBody->status, \ + pJoinBody->group_name); + return EFAULT; + } + } + } + + if ((result=tracker_mem_add_storage(pClientInfo, storage_id, ip_addr, \ + bNeedSleep, true, &bStorageInserted)) != 0) + { + return result; + } + + pStorageServer = pClientInfo->pStorage; + pStorageServer->store_path_count = pJoinBody->store_path_count; + pStorageServer->subdir_count_per_path = pJoinBody->subdir_count_per_path; + pStorageServer->upload_priority = pJoinBody->upload_priority; + pStorageServer->join_time = pJoinBody->join_time; + pStorageServer->up_time = pJoinBody->up_time; + snprintf(pStorageServer->version, sizeof(pStorageServer->version), \ + "%s", pJoinBody->version); + snprintf(pStorageServer->domain_name, \ + sizeof(pStorageServer->domain_name), \ + "%s", pJoinBody->domain_name); + pStorageServer->storage_port = pJoinBody->storage_port; + pStorageServer->storage_http_port = pJoinBody->storage_http_port; + + if (pClientInfo->pGroup->store_path_count == 0) + { + pClientInfo->pGroup->store_path_count = \ + pJoinBody->store_path_count; + if ((result=tracker_malloc_group_path_mbs( \ + pClientInfo->pGroup)) != 0) + { + return result; + } + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + if (pClientInfo->pGroup->store_path_count != \ + pJoinBody->store_path_count) + { + ppEnd = pClientInfo->pGroup->all_servers + \ + pClientInfo->pGroup->count; + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServerstore_path_count != \ + pJoinBody->store_path_count) + { + break; + } + } + + if (ppServer == ppEnd) //all servers are same, adjust + { + if ((result=tracker_realloc_group_path_mbs( \ + pClientInfo->pGroup, \ + pJoinBody->store_path_count))!=0) + { + return result; + } + + if ((result=tracker_save_groups()) != 0) + { + return result; + } + + logDebug("file: "__FILE__", line: %d, " \ + "all storage server's store_path_count " \ + "are same, adjust to %d", \ + __LINE__, pJoinBody->store_path_count); + } + else if (pJoinBody->store_path_count < \ + pClientInfo->pGroup->store_path_count) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, store_path_count %d less " \ + "than that of the group \"%s\", " \ + "the group store_path_count is %d", \ + __LINE__, ip_addr, \ + pJoinBody->store_path_count, \ + pJoinBody->group_name, \ + pClientInfo->pGroup->store_path_count); + return EINVAL; + } + } + } + + if (pClientInfo->pGroup->subdir_count_per_path == 0) + { + pClientInfo->pGroup->subdir_count_per_path = \ + pJoinBody->subdir_count_per_path; + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + if (pClientInfo->pGroup->subdir_count_per_path != \ + pJoinBody->subdir_count_per_path) + { + ppEnd = pClientInfo->pGroup->all_servers + \ + pClientInfo->pGroup->count; + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServersubdir_count_per_path != \ + pJoinBody->subdir_count_per_path) + { + break; + } + } + + if (ppServer == ppEnd) //all servers are same, adjust + { + pClientInfo->pGroup->subdir_count_per_path = \ + pJoinBody->subdir_count_per_path; + if ((result=tracker_save_groups()) != 0) + { + return result; + } + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, subdir_count_per_path %d is " \ + "not same in the group \"%s\", " \ + "group subdir_count_per_path is %d", \ + __LINE__, ip_addr, \ + pJoinBody->subdir_count_per_path, \ + pJoinBody->group_name,\ + pClientInfo->pGroup->subdir_count_per_path); + + return EINVAL; + } + } + } + + if (bStorageInserted) + { + if ((!pJoinBody->init_flag) && pJoinBody->status > 0) + { + if (pJoinBody->status == FDFS_STORAGE_STATUS_ACTIVE) + { + pStorageServer->status = FDFS_STORAGE_STATUS_ONLINE; + } + else + { + pStorageServer->status = pJoinBody->status; + } + } + + if ((result=tracker_save_sys_files()) != 0) + { + return result; + } + } + + if (pStorageServer->status == FDFS_STORAGE_STATUS_OFFLINE || \ + pStorageServer->status == FDFS_STORAGE_STATUS_RECOVERY) + { + pStorageServer->status = FDFS_STORAGE_STATUS_ONLINE; + } + else if (pStorageServer->status == FDFS_STORAGE_STATUS_INIT) + { + pStorageServer->changelog_offset = g_changelog_fsize; + } + + logDebug("file: "__FILE__", line: %d, " \ + "storage server %s::%s join in, remain changelog bytes: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + pClientInfo->pGroup->group_name, ip_addr, \ + g_changelog_fsize - pStorageServer->changelog_offset); + + return 0; +} + +int tracker_mem_sync_storages(FDFSGroupInfo *pGroup, \ + FDFSStorageBrief *briefServers, const int server_count) +{ + int result; + FDFSStorageBrief *pServer; + FDFSStorageBrief *pEnd; + FDFSStorageDetail target_storage; + FDFSStorageDetail *pTargetStorage; + FDFSStorageDetail **ppFound; + + if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + result = 0; + do + { + memset(&target_storage, 0, sizeof(target_storage)); + pEnd = briefServers + server_count; + for (pServer=briefServers; pServerid[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0'; + pServer->ip_addr[IP_ADDRESS_SIZE - 1] = '\0'; + if (pServer->status == FDFS_STORAGE_STATUS_NONE \ + || pServer->status == FDFS_STORAGE_STATUS_ACTIVE \ + || pServer->status == FDFS_STORAGE_STATUS_ONLINE) + { + continue; + } + + memcpy(target_storage.id, pServer->id, \ + FDFS_STORAGE_ID_MAX_SIZE); + pTargetStorage = &target_storage; + if ((ppFound=(FDFSStorageDetail **)bsearch( \ + &pTargetStorage, \ + pGroup->sorted_servers, \ + pGroup->count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id)) != NULL) + { + if ((*ppFound)->status == pServer->status \ + || (*ppFound)->status == \ + FDFS_STORAGE_STATUS_ONLINE \ + || (*ppFound)->status == \ + FDFS_STORAGE_STATUS_ACTIVE + || (*ppFound)->status == \ + FDFS_STORAGE_STATUS_RECOVERY) + { + continue; + } + + if (pServer->status == \ + FDFS_STORAGE_STATUS_DELETED + || pServer->status == \ + FDFS_STORAGE_STATUS_IP_CHANGED) + { + (*ppFound)->status = pServer->status; + pGroup->chg_count++; + continue; + } + + if (pServer->status > (*ppFound)->status) + { + (*ppFound)->status = pServer->status; + pGroup->chg_count++; + } + } + else if (pServer->status == FDFS_STORAGE_STATUS_DELETED + || pServer->status == FDFS_STORAGE_STATUS_IP_CHANGED) + { + //ignore deleted storage server + } + else + { + FDFSStorageDetail *pStorageServer; + bool bInserted; + + result = _tracker_mem_add_storage(pGroup, \ + &pStorageServer, pServer->id, \ + pServer->ip_addr, true, false, \ + &bInserted); + if (result != 0) + { + pStorageServer->status = pServer->status; + } + } + } + } while (0); + + if (pthread_mutex_unlock(&mem_thread_lock) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail", \ + __LINE__); + } + + return result; +} + +static void tracker_mem_find_store_server(FDFSGroupInfo *pGroup) +{ + if (pGroup->active_count == 0) + { + pGroup->pStoreServer = NULL; + return; + } + + if (g_groups.store_server == FDFS_STORE_SERVER_FIRST_BY_PRI) + { + FDFSStorageDetail **ppEnd; + FDFSStorageDetail **ppServer; + FDFSStorageDetail *pMinPriServer; + + pMinPriServer = *(pGroup->active_servers); + ppEnd = pGroup->active_servers + pGroup->active_count; + for (ppServer=pGroup->active_servers+1; ppServerupload_priority < \ + pMinPriServer->upload_priority) + { + pMinPriServer = *ppServer; + } + } + + pGroup->pStoreServer = pMinPriServer; + } + else + { + pGroup->pStoreServer = *(pGroup->active_servers); + } +} + +static int _storage_get_trunk_binlog_size( + ConnectionInfo *pStorageServer, int64_t *file_size) +{ + char out_buff[sizeof(TrackerHeader)]; + char in_buff[8]; + TrackerHeader *pHeader; + char *pInBuff; + int64_t in_bytes; + int result; + + pHeader = (TrackerHeader *)out_buff; + memset(out_buff, 0, sizeof(out_buff)); + pHeader->cmd = STORAGE_PROTO_CMD_TRUNK_GET_BINLOG_SIZE; + if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, \ + result, STRERROR(result)); + return result; + } + + pInBuff = in_buff; + if ((result=fdfs_recv_response(pStorageServer, \ + &pInBuff, sizeof(in_buff), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes != sizeof(in_buff)) + { + logError("file: "__FILE__", line: %d, " \ + "storage server %s:%d, recv body length: " \ + INT64_PRINTF_FORMAT" != %d", \ + __LINE__, pStorageServer->ip_addr, \ + pStorageServer->port, in_bytes, (int)sizeof(in_buff)); + return EINVAL; + } + + *file_size = buff2long(in_buff); + return 0; +} + +static int tracker_mem_get_trunk_binlog_size( + const char *storage_ip, const int port, int64_t *file_size) +{ + ConnectionInfo storage_server; + ConnectionInfo *conn; + int result; + + *file_size = 0; + strcpy(storage_server.ip_addr, storage_ip); + storage_server.port = port; + storage_server.sock = -1; + if ((conn=tracker_connect_server(&storage_server, &result)) == NULL) + { + return result; + } + + result = _storage_get_trunk_binlog_size(conn, file_size); + tracker_disconnect_server_ex(conn, result != 0); + + + logDebug("file: "__FILE__", line: %d, " \ + "storage %s:%d, trunk binlog file size: "INT64_PRINTF_FORMAT, \ + __LINE__, storage_server.ip_addr, storage_server.port, \ + *file_size); + return result; +} + +static int tracker_write_to_trunk_change_log(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pLastTrunkServer) +{ + char full_filename[MAX_PATH_SIZE]; + char buff[256]; + int fd; + int len; + struct tm tm; + time_t current_time; + FDFSStorageDetail *pLastTrunk; + + tracker_mem_file_lock(); + + snprintf(full_filename, sizeof(full_filename), "%s/logs/%s", \ + g_fdfs_base_path, TRUNK_SERVER_CHANGELOG_FILENAME); + if ((fd=open(full_filename, O_WRONLY | O_CREAT | O_APPEND, 0644)) < 0) + { + tracker_mem_file_unlock(); + logError("file: "__FILE__", line: %d, " \ + "open \"%s\" fail, errno: %d, error info: %s", \ + __LINE__, full_filename, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOENT; + } + + current_time = g_current_time; + localtime_r(¤t_time, &tm); + len = sprintf(buff, "[%04d-%02d-%02d %02d:%02d:%02d] %s ", \ + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \ + tm.tm_hour, tm.tm_min, tm.tm_sec, pGroup->group_name); + + pLastTrunk = pLastTrunkServer; + if (pLastTrunk == NULL && *(pGroup->last_trunk_server_id) != '\0') + { + pLastTrunk = tracker_mem_get_storage(pGroup, \ + pGroup->last_trunk_server_id); + } + if (g_use_storage_id) + { + if (pLastTrunk == NULL) + { + len += sprintf(buff + len, " %s/%s => ", \ + *(pGroup->last_trunk_server_id) == '\0' ? \ + "-" : pGroup->last_trunk_server_id, "-"); + } + else + { + len += sprintf(buff + len, " %s/%s => ", \ + pLastTrunk->id, pLastTrunk->ip_addr); + } + + if (pGroup->pTrunkServer == NULL) + { + len += sprintf(buff + len, " %s/%s\n", "-", "-"); + } + else + { + len += sprintf(buff + len, " %s/%s\n", \ + pGroup->pTrunkServer->id, \ + pGroup->pTrunkServer->ip_addr); + } + } + else + { + if (pLastTrunk == NULL) + { + len += sprintf(buff + len, " %s => ", \ + *(pGroup->last_trunk_server_id) == '\0' ? \ + "-" : pGroup->last_trunk_server_id); + } + else + { + len += sprintf(buff + len, " %s => ", \ + pLastTrunk->ip_addr); + } + + if (pGroup->pTrunkServer == NULL) + { + len += sprintf(buff + len, " %s\n", "-"); + } + else + { + len += sprintf(buff + len, " %s\n", \ + pGroup->pTrunkServer->ip_addr); + } + } + + if (write(fd, buff, len) != len) + { + logError("file: "__FILE__", line: %d, " \ + "write to file \"%s\" fail, " \ + "errno: %d, error info: %s", \ + __LINE__, full_filename, \ + errno, STRERROR(errno)); + } + + close(fd); + tracker_mem_file_unlock(); + + return 0; +} + +static int tracker_set_trunk_server_and_log(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pNewTrunkServer) +{ + FDFSStorageDetail *pLastTrunkServer; + + pLastTrunkServer = pGroup->pTrunkServer; + pGroup->pTrunkServer = pNewTrunkServer; + if (pNewTrunkServer == NULL || strcmp(pNewTrunkServer->id, \ + pGroup->last_trunk_server_id) != 0) + { + int result; + result = tracker_write_to_trunk_change_log(pGroup, \ + pLastTrunkServer); + if (pNewTrunkServer == NULL) + { + *(pGroup->last_trunk_server_id) = '\0'; + } + else + { + strcpy(pGroup->last_trunk_server_id, \ + pNewTrunkServer->id); + } + return result; + } + + return 0; +} + +static int tracker_mem_do_set_trunk_server(FDFSGroupInfo *pGroup, + FDFSStorageDetail *pTrunkServer, const bool save) +{ + int result; + + if (*(pGroup->last_trunk_server_id) != '\0' && + strcmp(pTrunkServer->id, pGroup->last_trunk_server_id) != 0) + { + if ((result=fdfs_deal_no_body_cmd_ex( + pTrunkServer->ip_addr, + pGroup->storage_port, + STORAGE_PROTO_CMD_TRUNK_DELETE_BINLOG_MARKS)) != 0) + { + return result; + } + } + + tracker_set_trunk_server_and_log(pGroup, pTrunkServer); + pGroup->trunk_chg_count++; + g_trunk_server_chg_count++; + + logInfo("file: "__FILE__", line: %d, " \ + "group: %s, trunk server set to %s(%s:%d)", __LINE__, \ + pGroup->group_name, pGroup->pTrunkServer->id, \ + pGroup->pTrunkServer->ip_addr, pGroup->storage_port); + if (save) + { + return tracker_save_groups(); + } + + return 0; +} + +static int tracker_mem_find_trunk_server(FDFSGroupInfo *pGroup, + const bool save) +{ + FDFSStorageDetail *pStoreServer; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + int result; + int64_t file_size; + int64_t max_file_size; + + pStoreServer = pGroup->pStoreServer; + if (pStoreServer == NULL) + { + return ENOENT; + } + + result = tracker_mem_get_trunk_binlog_size(pStoreServer->ip_addr, + pGroup->storage_port, &max_file_size); + if (result != 0) + { + return result; + } + + ppServerEnd = pGroup->active_servers + pGroup->active_count; + for (ppServer=pGroup->active_servers; ppServerip_addr, + pGroup->storage_port, &file_size); + if (result != 0) + { + continue; + } + + if (file_size > max_file_size) + { + pStoreServer = *ppServer; + } + } + + return tracker_mem_do_set_trunk_server(pGroup, \ + pStoreServer, save); +} + +const FDFSStorageDetail *tracker_mem_set_trunk_server( \ + FDFSGroupInfo *pGroup, const char *pStroageId, int *result) +{ + FDFSStorageDetail *pServer; + FDFSStorageDetail *pTrunkServer; + + if (!(g_if_leader_self && g_if_use_trunk_file)) + { + *result = EOPNOTSUPP; + return NULL; + } + + pTrunkServer = pGroup->pTrunkServer; + if (pStroageId == NULL || *pStroageId == '\0') + { + if (pTrunkServer != NULL && pTrunkServer-> \ + status == FDFS_STORAGE_STATUS_ACTIVE) + { + *result = 0; + return pTrunkServer; + } + + *result = tracker_mem_find_trunk_server(pGroup, true); + if (*result != 0) + { + return NULL; + } + return pGroup->pTrunkServer; + } + + if (pTrunkServer != NULL && pTrunkServer->status == \ + FDFS_STORAGE_STATUS_ACTIVE) + { + if (strcmp(pStroageId, pTrunkServer->id) == 0) + { + *result = EALREADY; + } + else + { + *result = EEXIST; + } + return pTrunkServer; + } + + pServer = tracker_mem_get_storage(pGroup, pStroageId); + if (pServer == NULL) + { + *result = ENOENT; + return NULL; + } + + if (pServer->status != FDFS_STORAGE_STATUS_ACTIVE) + { + *result = ENONET; + return NULL; + } + + *result = tracker_mem_do_set_trunk_server(pGroup, \ + pServer, true); + return *result == 0 ? pGroup->pTrunkServer : NULL; +} + +int tracker_mem_deactive_store_server(FDFSGroupInfo *pGroup, + FDFSStorageDetail *pTargetServer) +{ + int result; + FDFSStorageDetail **ppStorageServer; + FDFSStorageDetail **ppEnd; + FDFSStorageDetail **ppServer; + + if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + ppStorageServer = (FDFSStorageDetail **)bsearch( \ + &pTargetServer, \ + pGroup->active_servers, \ + pGroup->active_count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer != NULL) + { + (*ppStorageServer)->chg_count = 0; + (*ppStorageServer)->trunk_chg_count = 0; + + ppEnd = pGroup->active_servers + pGroup->active_count - 1; + for (ppServer=ppStorageServer; ppServeractive_count--; + pGroup->chg_count++; + +#ifdef WITH_HTTPD + if (g_http_check_interval <= 0) + { + pGroup->http_server_count = pGroup->active_count; + } +#endif + } + + tracker_mem_find_store_server(pGroup); + + if ((result=pthread_mutex_unlock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +int tracker_mem_active_store_server(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pTargetServer) +{ + int result; + FDFSStorageDetail **ppStorageServer; + + if ((pTargetServer->status == FDFS_STORAGE_STATUS_WAIT_SYNC) || \ + (pTargetServer->status == FDFS_STORAGE_STATUS_SYNCING) || \ + (pTargetServer->status == FDFS_STORAGE_STATUS_IP_CHANGED) || \ + (pTargetServer->status == FDFS_STORAGE_STATUS_INIT)) + { + return 0; + } + + /* + if (pTargetServer->status == FDFS_STORAGE_STATUS_DELETED) + { + logError("file: "__FILE__", line: %d, " \ + "storage ip: %s already deleted, you can " \ + "restart the tracker servers to reset.", \ + __LINE__, pTargetServer->ip_addr); + return EAGAIN; + } + */ + + pTargetServer->status = FDFS_STORAGE_STATUS_ACTIVE; + + ppStorageServer = (FDFSStorageDetail **)bsearch(&pTargetServer, \ + pGroup->active_servers, \ + pGroup->active_count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer != NULL) + { + return 0; + } + + if ((result=pthread_mutex_lock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + ppStorageServer = (FDFSStorageDetail **)bsearch(&pTargetServer, \ + pGroup->active_servers, \ + pGroup->active_count, \ + sizeof(FDFSStorageDetail *), \ + tracker_mem_cmp_by_storage_id); + if (ppStorageServer == NULL) + { + tracker_mem_insert_into_sorted_servers( \ + pTargetServer, pGroup->active_servers, \ + pGroup->active_count); + pGroup->active_count++; + pGroup->chg_count++; + +#ifdef WITH_HTTPD + if (g_http_check_interval <= 0) + { + pGroup->http_server_count = pGroup->active_count; + } +#endif + + if (g_use_storage_id) + { + logDebug("file: "__FILE__", line: %d, " \ + "storage server %s::%s(%s) now active", \ + __LINE__, pGroup->group_name, \ + pTargetServer->id, pTargetServer->ip_addr); + } + else + { + logDebug("file: "__FILE__", line: %d, " \ + "storage server %s::%s now active", \ + __LINE__, pGroup->group_name, \ + pTargetServer->ip_addr); + } + } + + tracker_mem_find_store_server(pGroup); + if (g_if_leader_self && g_if_use_trunk_file && \ + pGroup->pTrunkServer == NULL) + { + tracker_mem_find_trunk_server(pGroup, true); + } + + if ((result=pthread_mutex_unlock(&mem_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_unlock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + return 0; +} + +void tracker_mem_find_trunk_servers() +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + + if (!(g_if_leader_self && g_if_use_trunk_file)) + { + return; + } + + pthread_mutex_lock(&mem_thread_lock); + ppGroupEnd = g_groups.groups + g_groups.count; + for (ppGroup=g_groups.groups; ppGrouppTrunkServer == NULL) + { + tracker_mem_find_trunk_server(*ppGroup, true); + } + } + pthread_mutex_unlock(&mem_thread_lock); +} + +int tracker_mem_offline_store_server(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pStorage) +{ + pStorage->up_time = 0; + if ((pStorage->status == FDFS_STORAGE_STATUS_WAIT_SYNC) || \ + (pStorage->status == FDFS_STORAGE_STATUS_SYNCING) || \ + (pStorage->status == FDFS_STORAGE_STATUS_INIT) || \ + (pStorage->status == FDFS_STORAGE_STATUS_DELETED) || \ + (pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED) || \ + (pStorage->status == FDFS_STORAGE_STATUS_RECOVERY)) + { + return 0; + } + + if (g_use_storage_id) + { + logDebug("file: "__FILE__", line: %d, " \ + "storage server %s::%s (%s) offline", \ + __LINE__, pGroup->group_name, \ + pStorage->id, pStorage->ip_addr); + } + else + { + logDebug("file: "__FILE__", line: %d, " \ + "storage server %s::%s offline", \ + __LINE__, pGroup->group_name, \ + pStorage->ip_addr); + } + + pStorage->status = FDFS_STORAGE_STATUS_OFFLINE; + return tracker_mem_deactive_store_server(pGroup, pStorage); +} + +FDFSStorageDetail *tracker_get_writable_storage(FDFSGroupInfo *pStoreGroup) +{ + int write_server_index; + if (g_groups.store_server == FDFS_STORE_SERVER_ROUND_ROBIN) + { + write_server_index = pStoreGroup->current_write_server++; + if (pStoreGroup->current_write_server >= \ + pStoreGroup->active_count) + { + pStoreGroup->current_write_server = 0; + } + + if (write_server_index >= pStoreGroup->active_count) + { + write_server_index = 0; + } + return *(pStoreGroup->active_servers + write_server_index); + } + else //use the first server + { + return pStoreGroup->pStoreServer; + } +} + +int tracker_mem_get_storage_by_filename(const byte cmd,FDFS_DOWNLOAD_TYPE_PARAM\ + const char *group_name, const char *filename, const int filename_len, \ + FDFSGroupInfo **ppGroup, FDFSStorageDetail **ppStoreServers, \ + int *server_count) +{ + char szIpAddr[IP_ADDRESS_SIZE]; + char storage_id[FDFS_STORAGE_ID_MAX_SIZE]; + FDFSStorageDetail *pStoreSrcServer; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + FDFSStorageDetail *pGroupStoreServer; + int file_timestamp; + int storage_ip; + int read_server_index; + int cmp_res; + struct in_addr ip_addr; + time_t current_time; + bool bNormalFile; + + *server_count = 0; + *ppGroup = tracker_mem_get_group(group_name); + if (*ppGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "invalid group_name: %s", \ + __LINE__, group_name); + return ENOENT; + } + +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + if ((*ppGroup)->active_count == 0) + { + return ENOENT; + } + } + else + { + if ((*ppGroup)->http_server_count == 0) + { + return ENOENT; + } + } +#else + if ((*ppGroup)->active_count == 0) + { + return ENOENT; + } +#endif + + pGroupStoreServer = (*ppGroup)->pStoreServer; + if (pGroupStoreServer == NULL) + { + return ENOENT; + } + + //file generated by version < v1.12 + if (filename_len < 32 + (FDFS_FILE_EXT_NAME_MAX_LEN + 1)) + { + storage_ip = INADDR_NONE; + *storage_id = '\0'; + file_timestamp = 0; + bNormalFile = true; + } + else //file generated by version >= v1.12 + { + int64_t file_size; + + char name_buff[64]; + int decoded_len; + + base64_decode_auto(&g_base64_context, (char *)filename + \ + FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \ + name_buff, &decoded_len); + storage_ip = ntohl(buff2int(name_buff)); + file_timestamp = buff2int(name_buff+sizeof(int)); + file_size = buff2long(name_buff + sizeof (int) * 2); + + if (fdfs_get_server_id_type(storage_ip) == FDFS_ID_TYPE_SERVER_ID) + { + sprintf(storage_id, "%d", storage_ip); + } + else + { + *storage_id = '\0'; + } + + bNormalFile = !(IS_SLAVE_FILE(filename_len, file_size) || \ + IS_APPENDER_FILE(file_size)); + } + + /* + //logInfo("cmd=%d, bNormalFile=%d, storage_ip=%d, storage_id=%s, " + "file_timestamp=%d\n", cmd, bNormalFile, storage_ip, + storage_id, file_timestamp); + */ + + memset(szIpAddr, 0, sizeof(szIpAddr)); + if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE) + { + if (g_groups.download_server == \ + FDFS_DOWNLOAD_SERVER_SOURCE_FIRST) + { +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_storage_by_id(\ + *ppGroup, storage_id); + } + else + { + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = storage_ip; + pStoreSrcServer=tracker_mem_get_active_storage_by_ip( \ + *ppGroup, inet_ntop(AF_INET, &ip_addr, \ + szIpAddr, sizeof(szIpAddr))); + } + } + else + { + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_http_server_by_id( \ + *ppGroup, storage_id); + } + else + { + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = storage_ip; + pStoreSrcServer=tracker_mem_get_active_http_server_by_ip( \ + *ppGroup, inet_ntop(AF_INET, &ip_addr, \ + szIpAddr, sizeof(szIpAddr))); + } + } +#else + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_storage_by_id(\ + *ppGroup, storage_id); + } + else + { + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = storage_ip; + pStoreSrcServer=tracker_mem_get_active_storage_by_ip(\ + *ppGroup, inet_ntop(AF_INET, &ip_addr, \ + szIpAddr, sizeof(szIpAddr))); + } +#endif + if (pStoreSrcServer != NULL) + { + ppStoreServers[(*server_count)++] = \ + pStoreSrcServer; + return 0; + } + } + + //round robin +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + read_server_index = (*ppGroup)->current_read_server; + if (read_server_index >= (*ppGroup)->active_count) + { + read_server_index = 0; + } + ppStoreServers[(*server_count)++]=*((*ppGroup)->active_servers \ + + read_server_index); + } + else + { + read_server_index = (*ppGroup)->current_http_server; + if (read_server_index >= (*ppGroup)->http_server_count) + { + read_server_index = 0; + } + ppStoreServers[(*server_count)++]=*((*ppGroup)->http_servers \ + + read_server_index); + } +#else + read_server_index = (*ppGroup)->current_read_server; + if (read_server_index >= (*ppGroup)->active_count) + { + read_server_index = 0; + } + ppStoreServers[(*server_count)++]=*((*ppGroup)->active_servers \ + + read_server_index); +#endif + /* + //logInfo("filename=%s, storage server ip=%s, " \ + "file_timestamp=%d, last_synced_timestamp=%d\n", + filename, ppStoreServers[0]->ip_addr, file_timestamp, \ + (int)ppStoreServers[0]->stat.last_synced_timestamp); + */ + + do + { + if (bNormalFile) + { + current_time = g_current_time; + if ((file_timestamp < current_time - \ + g_storage_sync_file_max_delay) || \ + (ppStoreServers[0]->stat.last_synced_timestamp > \ + file_timestamp) || \ + (ppStoreServers[0]->stat.last_synced_timestamp + 1 >= \ + file_timestamp && current_time - file_timestamp > \ + g_storage_sync_file_max_time)\ + || (storage_ip == INADDR_NONE \ + && g_groups.store_server == FDFS_STORE_SERVER_ROUND_ROBIN)) + { + break; + } + + if (storage_ip == INADDR_NONE) + { +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + ppStoreServers[0] = pGroupStoreServer; + break; + } + else + { + pStoreSrcServer=tracker_mem_get_active_storage_by_id( + *ppGroup, pGroupStoreServer->id); + if (pStoreSrcServer != NULL) + { + ppStoreServers[0] = pStoreSrcServer; + break; + } + } +#else + ppStoreServers[0] = pGroupStoreServer; + break; +#endif + } + } + + memset(&ip_addr, 0, sizeof(ip_addr)); + if (*storage_id != '\0') + { + cmp_res = strcmp(storage_id, ppStoreServers[0]->id); + } + else + { + ip_addr.s_addr = storage_ip; + inet_ntop(AF_INET, &ip_addr, szIpAddr, sizeof(szIpAddr)); + cmp_res = strcmp(szIpAddr, ppStoreServers[0]->ip_addr); + } + if (cmp_res == 0) + { +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + break; + } + else //http + { + if (*storage_id != '\0') + { + if (tracker_mem_get_active_http_server_by_id( + *ppGroup, storage_id) != NULL) + { + break; + } + } + else if (tracker_mem_get_active_http_server_by_ip( + *ppGroup, szIpAddr) != NULL) + { + break; + } + } +#else + break; +#endif + } + + if (g_groups.download_server == \ + FDFS_DOWNLOAD_SERVER_ROUND_ROBIN) + { //avoid search again +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_storage_by_id( + *ppGroup, storage_id); + } + else + { + pStoreSrcServer=tracker_mem_get_active_storage_by_ip( + *ppGroup, szIpAddr); + } + } + else //http + { + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_http_server_by_id( + *ppGroup, storage_id); + } + else + { + pStoreSrcServer=tracker_mem_get_active_http_server_by_ip( + *ppGroup, szIpAddr); + } + } +#else + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_storage_by_id( + *ppGroup, storage_id); + } + else + { + pStoreSrcServer=tracker_mem_get_active_storage_by_ip( + *ppGroup, szIpAddr); + } +#endif + if (pStoreSrcServer != NULL) + { + ppStoreServers[0] = pStoreSrcServer; + break; + } + } + + if (g_groups.store_server != \ + FDFS_STORE_SERVER_ROUND_ROBIN) + { +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + ppStoreServers[0] = pGroupStoreServer; + } + else //http + { + pStoreSrcServer=tracker_mem_get_active_http_server_by_id( + *ppGroup, pGroupStoreServer->id); + if (pStoreSrcServer != NULL) + { + ppStoreServers[0] = pStoreSrcServer; + } + else + { + ppStoreServers[0] = *((*ppGroup)->http_servers); + } + } +#else + ppStoreServers[0] = pGroupStoreServer; +#endif + break; + } + } while (0); + +#ifdef WITH_HTTPD + if (download_type == FDFS_DOWNLOAD_TYPE_TCP) + { + (*ppGroup)->current_read_server++; + if ((*ppGroup)->current_read_server >= (*ppGroup)->active_count) + { + (*ppGroup)->current_read_server = 0; + } + } + else //http + { + (*ppGroup)->current_http_server++; + if ((*ppGroup)->current_http_server >= \ + (*ppGroup)->http_server_count) + { + (*ppGroup)->current_http_server = 0; + } + } +#else + (*ppGroup)->current_read_server++; + if ((*ppGroup)->current_read_server >= (*ppGroup)->active_count) + { + (*ppGroup)->current_read_server = 0; + } +#endif + } + else if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE) + { + if (storage_ip != INADDR_NONE) + { + if (*storage_id != '\0') + { + pStoreSrcServer=tracker_mem_get_active_storage_by_id(\ + *ppGroup, storage_id); + } + else + { + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = storage_ip; + pStoreSrcServer=tracker_mem_get_active_storage_by_ip(\ + *ppGroup, inet_ntop(AF_INET, &ip_addr, \ + szIpAddr, sizeof(szIpAddr))); + } + + if (pStoreSrcServer != NULL) + { + ppStoreServers[(*server_count)++] = \ + pStoreSrcServer; + return 0; + } + } + + ppStoreServers[0] = tracker_get_writable_storage(*ppGroup); + *server_count = ppStoreServers[0] != NULL ? 1 : 0; + } + else //TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL + { + memset(szIpAddr, 0, sizeof(szIpAddr)); + if (storage_ip != INADDR_NONE) + { + memset(&ip_addr, 0, sizeof(ip_addr)); + ip_addr.s_addr = storage_ip; + inet_ntop(AF_INET, &ip_addr,szIpAddr,sizeof(szIpAddr)); + } + + if (bNormalFile) + { + current_time = g_current_time; + ppServerEnd = (*ppGroup)->active_servers + \ + (*ppGroup)->active_count; + + for (ppServer=(*ppGroup)->active_servers; \ + ppServerstat.last_synced_timestamp > \ + file_timestamp) || \ + ((*ppServer)->stat.last_synced_timestamp + 1 >= \ + file_timestamp && current_time - file_timestamp > \ + g_storage_sync_file_max_time) \ + || (storage_ip == INADDR_NONE \ + && g_groups.store_server == \ + FDFS_STORE_SERVER_ROUND_ROBIN) + || strcmp((*ppServer)->ip_addr, szIpAddr) == 0) + { + ppStoreServers[(*server_count)++] = *ppServer; + } + } + } + else + { + if (*storage_id != '\0') + { + pStoreSrcServer = tracker_mem_get_active_storage_by_id( \ + *ppGroup, storage_id); + } + else + { + pStoreSrcServer = tracker_mem_get_active_storage_by_ip( \ + *ppGroup, szIpAddr); + } + + if (pStoreSrcServer != NULL) + { + ppStoreServers[(*server_count)++] = \ + pStoreSrcServer; + } + } + + if (*server_count == 0) + { + ppStoreServers[(*server_count)++] = pGroupStoreServer; + } + } + + return *server_count > 0 ? 0 : ENOENT; +} + +int tracker_mem_check_alive(void *arg) +{ + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + FDFSStorageDetail *deactiveServers[FDFS_MAX_SERVERS_EACH_GROUP]; + int deactiveCount; + time_t current_time; + + current_time = g_current_time; + ppGroupEnd = g_groups.groups + g_groups.count; + for (ppGroup=g_groups.groups; ppGroupactive_servers + (*ppGroup)->active_count; + for (ppServer=(*ppGroup)->active_servers; ppServerstat.last_heart_beat_time > \ + g_check_active_interval) + { + deactiveServers[deactiveCount] = *ppServer; + deactiveCount++; + if (deactiveCount >= FDFS_MAX_SERVERS_EACH_GROUP) + { + break; + } + } + } + + if (deactiveCount == 0) + { + continue; + } + + ppServerEnd = deactiveServers + deactiveCount; + for (ppServer=deactiveServers; ppServerstatus = FDFS_STORAGE_STATUS_OFFLINE; + tracker_mem_deactive_store_server(*ppGroup, *ppServer); + if (g_use_storage_id) + { + logInfo("file: "__FILE__", line: %d, " \ + "storage server %s(%s:%d) idle too long, " \ + "status change to offline!", __LINE__, \ + (*ppServer)->id, (*ppServer)->ip_addr, \ + (*ppGroup)->storage_port); + } + else + { + logInfo("file: "__FILE__", line: %d, " \ + "storage server %s:%d idle too long, " \ + "status change to offline!", __LINE__, \ + (*ppServer)->ip_addr, (*ppGroup)->storage_port); + } + } + } + + if ((!g_if_leader_self) || (!g_if_use_trunk_file)) + { + return 0; + } + + for (ppGroup=g_groups.groups; ppGrouppTrunkServer != NULL) + { + int check_trunk_times; + int check_trunk_interval; + int last_beat_interval; + + if (current_time - (*ppGroup)->pTrunkServer->up_time <= \ + 10 * g_check_active_interval) + { + if (g_trunk_init_check_occupying) + { + check_trunk_times = 5; + } + else + { + check_trunk_times = 3; + } + + if (g_trunk_init_reload_from_binlog) + { + check_trunk_times *= 2; + } + } + else + { + check_trunk_times = 2; + } + + last_beat_interval = current_time - (*ppGroup)-> \ + pTrunkServer->stat.last_heart_beat_time; + check_trunk_interval = check_trunk_times * \ + g_check_active_interval; + if (last_beat_interval > check_trunk_interval) + { + logInfo("file: "__FILE__", line: %d, " \ + "trunk server %s(%s:%d) offline because idle " \ + "time: %d s > threshold: %d s, should " \ + "re-select trunk server", __LINE__, \ + (*ppGroup)->pTrunkServer->id, \ + (*ppGroup)->pTrunkServer->ip_addr, \ + (*ppGroup)->storage_port, last_beat_interval, \ + check_trunk_interval); + + (*ppGroup)->pTrunkServer = NULL; + tracker_mem_find_trunk_server(*ppGroup, false); + if ((*ppGroup)->pTrunkServer == NULL) + { + tracker_set_trunk_server_and_log(*ppGroup, NULL); + } + + (*ppGroup)->trunk_chg_count++; + g_trunk_server_chg_count++; + + tracker_save_groups(); + } + } + else + { + tracker_mem_find_trunk_server(*ppGroup, true); + } + } + + return 0; +} + +int tracker_mem_get_storage_index(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pStorage) +{ + FDFSStorageDetail **ppStorage; + FDFSStorageDetail **ppEnd; + + ppEnd = pGroup->all_servers + pGroup->count; + for (ppStorage=pGroup->all_servers; ppStorageall_servers; + } + } + + logError("file: "__FILE__", line: %d, " \ + "get index of storage %s fail!!!", \ + __LINE__, pStorage->ip_addr); + + return -1; +} + diff --git a/tracker/tracker_mem.h b/tracker/tracker_mem.h new file mode 100644 index 0000000..5b47916 --- /dev/null +++ b/tracker/tracker_mem.h @@ -0,0 +1,137 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_mem.h + +#ifndef _TRACKER_MEM_H_ +#define _TRACKER_MEM_H_ + +#include +#include +#include "tracker_types.h" + +#define TRACKER_SYS_FILE_COUNT 4 +#define STORAGE_GROUPS_LIST_FILENAME_OLD "storage_groups.dat" +#define STORAGE_GROUPS_LIST_FILENAME_NEW "storage_groups_new.dat" +#define STORAGE_SERVERS_LIST_FILENAME_OLD "storage_servers.dat" +#define STORAGE_SERVERS_LIST_FILENAME_NEW "storage_servers_new.dat" +#define STORAGE_SERVERS_CHANGELOG_FILENAME "storage_changelog.dat" +#define STORAGE_SYNC_TIMESTAMP_FILENAME "storage_sync_timestamp.dat" +#define TRUNK_SERVER_CHANGELOG_FILENAME "trunk_server_change.log" +#define STORAGE_DATA_FIELD_SEPERATOR ',' + +typedef struct { + ConnectionInfo *pTrackerServer; + int running_time; //running seconds, more means higher weight + int restart_interval; //restart interval, less mean higher weight + bool if_leader; //if leader +} TrackerRunningStatus; + +#ifdef __cplusplus +extern "C" { +#endif + +extern TrackerServerGroup g_tracker_servers; //save all tracker servers from storage server +extern ConnectionInfo *g_last_tracker_servers; //for delay free +extern int g_next_leader_index; //next leader index +extern int g_tracker_leader_chg_count; //for notify storage servers +extern int g_trunk_server_chg_count; //for notify other trackers + +extern int64_t g_changelog_fsize; //storage server change log file size +extern char *g_tracker_sys_filenames[TRACKER_SYS_FILE_COUNT]; + +int tracker_mem_init(); +int tracker_mem_destroy(); + +int tracker_mem_init_pthread_lock(pthread_mutex_t *pthread_lock); +int tracker_mem_pthread_lock(); +int tracker_mem_pthread_unlock(); + +int tracker_mem_file_lock(); +int tracker_mem_file_unlock(); + +#define tracker_mem_get_group(group_name) \ + tracker_mem_get_group_ex((&g_groups), group_name) + +FDFSGroupInfo *tracker_mem_get_group_ex(FDFSGroups *pGroups, \ + const char *group_name); + +FDFSStorageDetail *tracker_mem_get_storage(FDFSGroupInfo *pGroup, \ + const char *id); +FDFSStorageDetail *tracker_mem_get_storage_by_ip(FDFSGroupInfo *pGroup, \ + const char *ip_addr); + +const FDFSStorageDetail *tracker_mem_set_trunk_server( \ + FDFSGroupInfo *pGroup, const char *pStroageId, int *result); +int tracker_mem_delete_storage(FDFSGroupInfo *pGroup, const char *id); + +int tracker_mem_storage_ip_changed(FDFSGroupInfo *pGroup, \ + const char *old_storage_ip, const char *new_storage_ip); + +int tracker_mem_add_group_and_storage(TrackerClientInfo *pClientInfo, \ + const char *ip_addr, FDFSStorageJoinBody *pJoinBody, \ + const bool bNeedSleep); + +int tracker_mem_offline_store_server(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pStorage); +int tracker_mem_active_store_server(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pTargetServer); + +int tracker_mem_sync_storages(FDFSGroupInfo *pGroup, \ + FDFSStorageBrief *briefServers, const int server_count); +int tracker_save_storages(); +int tracker_save_sync_timestamps(); +int tracker_save_sys_files(); + +int tracker_get_group_file_count(FDFSGroupInfo *pGroup); +int tracker_get_group_success_upload_count(FDFSGroupInfo *pGroup); +FDFSStorageDetail *tracker_get_group_sync_src_server(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pDestServer); + +FDFSStorageDetail *tracker_get_writable_storage(FDFSGroupInfo *pStoreGroup); + +#ifdef WITH_HTTPD +#define FDFS_DOWNLOAD_TYPE_PARAM const int download_type, +#define FDFS_DOWNLOAD_TYPE_CALL FDFS_DOWNLOAD_TYPE_TCP, +#else +#define FDFS_DOWNLOAD_TYPE_PARAM +#define FDFS_DOWNLOAD_TYPE_CALL +#endif + +int tracker_mem_get_storage_by_filename(const byte cmd,FDFS_DOWNLOAD_TYPE_PARAM\ + const char *group_name, const char *filename, const int filename_len, \ + FDFSGroupInfo **ppGroup, FDFSStorageDetail **ppStoreServers, \ + int *server_count); + +int tracker_mem_check_alive(void *arg); + +int tracker_mem_get_storage_index(FDFSGroupInfo *pGroup, \ + FDFSStorageDetail *pStorage); + +#define tracker_get_sys_files_start(pTrackerServer) \ + fdfs_deal_no_body_cmd(pTrackerServer, \ + TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_START) + +#define tracker_get_sys_files_end(pTrackerServer) \ + fdfs_deal_no_body_cmd(pTrackerServer, \ + TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_END) + +void tracker_calc_running_times(TrackerRunningStatus *pStatus); + +int tracker_mem_get_status(ConnectionInfo *pTrackerServer, \ + TrackerRunningStatus *pStatus); + +int tracker_save_groups(); + +void tracker_mem_find_trunk_servers(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_nio.c b/tracker/tracker_nio.c new file mode 100644 index 0000000..2d52512 --- /dev/null +++ b/tracker/tracker_nio.c @@ -0,0 +1,422 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "shared_func.h" +#include "sched_thread.h" +#include "fdfs_global.h" +#include "logger.h" +#include "sockopt.h" +#include "fast_task_queue.h" +#include "tracker_types.h" +#include "tracker_proto.h" +#include "tracker_mem.h" +#include "tracker_global.h" +#include "tracker_service.h" +#include "ioevent_loop.h" +#include "tracker_nio.h" + +static void client_sock_read(int sock, short event, void *arg); +static void client_sock_write(int sock, short event, void *arg); + +void task_finish_clean_up(struct fast_task_info *pTask) +{ + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + if (pTask->finish_callback != NULL) + { + pTask->finish_callback(pTask); + pTask->finish_callback = NULL; + } + + if (pClientInfo->pGroup != NULL) + { + if (pClientInfo->pStorage != NULL) + { + tracker_mem_offline_store_server(pClientInfo->pGroup, \ + pClientInfo->pStorage); + } + } + + ioevent_detach(&pTask->thread_data->ev_puller, pTask->event.fd); + close(pTask->event.fd); + pTask->event.fd = -1; + + if (pTask->event.timer.expires > 0) + { + fast_timer_remove(&pTask->thread_data->timer, + &pTask->event.timer); + pTask->event.timer.expires = 0; + } + + memset(pTask->arg, 0, sizeof(TrackerClientInfo)); + free_queue_push(pTask); +} + +void recv_notify_read(int sock, short event, void *arg) +{ + int bytes; + int incomesock; + struct nio_thread_data *pThreadData; + struct fast_task_info *pTask; + char szClientIp[IP_ADDRESS_SIZE]; + in_addr_t client_addr; + + while (1) + { + if ((bytes=read(sock, &incomesock, sizeof(incomesock))) < 0) + { + if (!(errno == EAGAIN || errno == EWOULDBLOCK)) + { + logError("file: "__FILE__", line: %d, " \ + "call read failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + + break; + } + else if (bytes == 0) + { + break; + } + + if (incomesock < 0) + { + return; + } + + client_addr = getPeerIpaddr(incomesock, \ + szClientIp, IP_ADDRESS_SIZE); + if (g_allow_ip_count >= 0) + { + if (bsearch(&client_addr, g_allow_ip_addrs, \ + g_allow_ip_count, sizeof(in_addr_t), \ + cmp_by_ip_addr_t) == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "ip addr %s is not allowed to access", \ + __LINE__, szClientIp); + + close(incomesock); + continue; + } + } + + if (tcpsetnonblockopt(incomesock) != 0) + { + close(incomesock); + continue; + } + + pTask = free_queue_pop(); + if (pTask == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc task buff failed, you should " \ + "increase the parameter: max_connections", \ + __LINE__); + close(incomesock); + continue; + } + + strcpy(pTask->client_ip, szClientIp); + + pThreadData = g_thread_data + incomesock % g_work_threads; + if (ioevent_set(pTask, pThreadData, incomesock, IOEVENT_READ, + client_sock_read, g_fdfs_network_timeout) != 0) + { + task_finish_clean_up(pTask); + continue; + } + } +} + +static int set_send_event(struct fast_task_info *pTask) +{ + int result; + + if (pTask->event.callback == client_sock_write) + { + return 0; + } + + pTask->event.callback = client_sock_write; + if (ioevent_modify(&pTask->thread_data->ev_puller, + pTask->event.fd, IOEVENT_WRITE, pTask) != 0) + { + result = errno != 0 ? errno : ENOENT; + task_finish_clean_up(pTask); + + logError("file: "__FILE__", line: %d, "\ + "ioevent_modify fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + return 0; +} + +int send_add_event(struct fast_task_info *pTask) +{ + pTask->offset = 0; + + /* direct send */ + client_sock_write(pTask->event.fd, IOEVENT_WRITE, pTask); + return 0; +} + +static void client_sock_read(int sock, short event, void *arg) +{ + int bytes; + int recv_bytes; + struct fast_task_info *pTask; + + pTask = (struct fast_task_info *)arg; + + if (event & IOEVENT_TIMEOUT) + { + if (pTask->offset == 0 && pTask->req_count > 0) + { + pTask->event.timer.expires = g_current_time + + g_fdfs_network_timeout; + fast_timer_add(&pTask->thread_data->timer, + &pTask->event.timer); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv timeout, " \ + "recv offset: %d, expect length: %d", \ + __LINE__, pTask->client_ip, \ + pTask->offset, pTask->length); + + task_finish_clean_up(pTask); + } + + return; + } + + if (event & IOEVENT_ERROR) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv error event: %d, " + "close connection", __LINE__, pTask->client_ip, event); + + task_finish_clean_up(pTask); + return; + } + + while (1) + { + fast_timer_modify(&pTask->thread_data->timer, + &pTask->event.timer, g_current_time + + g_fdfs_network_timeout); + if (pTask->length == 0) //recv header + { + recv_bytes = sizeof(TrackerHeader) - pTask->offset; + } + else + { + recv_bytes = pTask->length - pTask->offset; + } + + bytes = recv(sock, pTask->data + pTask->offset, recv_bytes, 0); + if (bytes < 0) + { + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv failed, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + errno, STRERROR(errno)); + + task_finish_clean_up(pTask); + } + + return; + } + else if (bytes == 0) + { + logDebug("file: "__FILE__", line: %d, " \ + "client ip: %s, recv failed, " \ + "connection disconnected.", \ + __LINE__, pTask->client_ip); + + task_finish_clean_up(pTask); + return; + } + + if (pTask->length == 0) //header + { + if (pTask->offset + bytes < sizeof(TrackerHeader)) + { + pTask->offset += bytes; + return; + } + + pTask->length = buff2long(((TrackerHeader *) \ + pTask->data)->pkg_len); + if (pTask->length < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length: %d < 0", \ + __LINE__, pTask->client_ip, \ + pTask->length); + + task_finish_clean_up(pTask); + return; + } + + pTask->length += sizeof(TrackerHeader); + if (pTask->length > TRACKER_MAX_PACKAGE_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, pkg length: %d > " \ + "max pkg size: %d", __LINE__, \ + pTask->client_ip, pTask->length, \ + TRACKER_MAX_PACKAGE_SIZE); + + task_finish_clean_up(pTask); + return; + } + } + + pTask->offset += bytes; + if (pTask->offset >= pTask->length) //recv done + { + pTask->req_count++; + tracker_deal_task(pTask); + return; + } + } + + return; +} + +static void client_sock_write(int sock, short event, void *arg) +{ + int bytes; + int result; + struct fast_task_info *pTask; + + pTask = (struct fast_task_info *)arg; + if (event & IOEVENT_TIMEOUT) + { + logError("file: "__FILE__", line: %d, " \ + "send timeout", __LINE__); + + task_finish_clean_up(pTask); + + return; + } + + if (event & IOEVENT_ERROR) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv error event: %d, " + "close connection", __LINE__, pTask->client_ip, event); + + task_finish_clean_up(pTask); + return; + } + + while (1) + { + fast_timer_modify(&pTask->thread_data->timer, + &pTask->event.timer, g_current_time + + g_fdfs_network_timeout); + + bytes = send(sock, pTask->data + pTask->offset, \ + pTask->length - pTask->offset, 0); + //printf("%08X sended %d bytes\n", (int)pTask, bytes); + if (bytes < 0) + { + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + set_send_event(pTask); + } + else + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, recv failed, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + errno, STRERROR(errno)); + + task_finish_clean_up(pTask); + } + + return; + } + else if (bytes == 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "send failed, connection disconnected.", \ + __LINE__); + + task_finish_clean_up(pTask); + return; + } + + pTask->offset += bytes; + if (pTask->offset >= pTask->length) + { + if (pTask->length == sizeof(TrackerHeader) && \ + ((TrackerHeader *)pTask->data)->status == EINVAL) + { + logDebug("file: "__FILE__", line: %d, "\ + "close conn: #%d, client ip: %s", \ + __LINE__, pTask->event.fd, + pTask->client_ip); + task_finish_clean_up(pTask); + return; + } + + pTask->offset = 0; + pTask->length = 0; + + pTask->event.callback = client_sock_read; + if (ioevent_modify(&pTask->thread_data->ev_puller, + pTask->event.fd, IOEVENT_READ, pTask) != 0) + { + result = errno != 0 ? errno : ENOENT; + task_finish_clean_up(pTask); + + logError("file: "__FILE__", line: %d, "\ + "ioevent_modify fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return; + } + + return; + } + } +} + diff --git a/tracker/tracker_nio.h b/tracker/tracker_nio.h new file mode 100644 index 0000000..13e1470 --- /dev/null +++ b/tracker/tracker_nio.h @@ -0,0 +1,33 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_nio.h + +#ifndef _TRACKER_NIO_H +#define _TRACKER_NIO_H + +#include +#include +#include +#include "fast_task_queue.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void recv_notify_read(int sock, short event, void *arg); +int send_add_event(struct fast_task_info *pTask); + +void task_finish_clean_up(struct fast_task_info *pTask); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/tracker/tracker_proto.c b/tracker/tracker_proto.c new file mode 100644 index 0000000..372eed4 --- /dev/null +++ b/tracker/tracker_proto.c @@ -0,0 +1,619 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "shared_func.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "tracker_types.h" +#include "tracker_proto.h" + +int fdfs_recv_header(ConnectionInfo *pTrackerServer, int64_t *in_bytes) +{ + TrackerHeader resp; + int result; + + if ((result=tcprecvdata_nb(pTrackerServer->sock, &resp, \ + sizeof(resp), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + *in_bytes = 0; + return result; + } + + if (resp.status != 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, response status %d != 0", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, resp.status); + + *in_bytes = 0; + return resp.status; + } + + *in_bytes = buff2long(resp.pkg_len); + if (*in_bytes < 0) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv package size " \ + INT64_PRINTF_FORMAT" is not correct", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, *in_bytes); + *in_bytes = 0; + return EINVAL; + } + + return resp.status; +} + +int fdfs_recv_response(ConnectionInfo *pTrackerServer, \ + char **buff, const int buff_size, \ + int64_t *in_bytes) +{ + int result; + bool bMalloced; + + result = fdfs_recv_header(pTrackerServer, in_bytes); + if (result != 0) + { + return result; + } + + if (*in_bytes == 0) + { + return 0; + } + + if (*buff == NULL) + { + *buff = (char *)malloc((*in_bytes) + 1); + if (*buff == NULL) + { + *in_bytes = 0; + + logError("file: "__FILE__", line: %d, " \ + "malloc "INT64_PRINTF_FORMAT" bytes fail", \ + __LINE__, (*in_bytes) + 1); + return errno != 0 ? errno : ENOMEM; + } + + bMalloced = true; + } + else + { + if (*in_bytes > buff_size) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv body bytes: " \ + INT64_PRINTF_FORMAT" exceed max: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, *in_bytes, buff_size); + *in_bytes = 0; + return ENOSPC; + } + + bMalloced = false; + } + + if ((result=tcprecvdata_nb(pTrackerServer->sock, *buff, \ + *in_bytes, g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server: %s:%d, recv data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + *in_bytes = 0; + if (bMalloced) + { + free(*buff); + *buff = NULL; + } + return result; + } + + return 0; +} + +int fdfs_quit(ConnectionInfo *pTrackerServer) +{ + TrackerHeader header; + int result; + + memset(&header, 0, sizeof(header)); + header.cmd = FDFS_PROTO_CMD_QUIT; + result = tcpsenddata_nb(pTrackerServer->sock, &header, \ + sizeof(header), g_fdfs_network_timeout); + if(result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server: %s:%d, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + return 0; +} + +int fdfs_deal_no_body_cmd(ConnectionInfo *pTrackerServer, const int cmd) +{ + TrackerHeader header; + int result; + int64_t in_bytes; + + memset(&header, 0, sizeof(header)); + header.cmd = cmd; + result = tcpsenddata_nb(pTrackerServer->sock, &header, \ + sizeof(header), g_fdfs_network_timeout); + if(result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server ip: %s, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + result, STRERROR(result)); + return result; + } + + result = fdfs_recv_header(pTrackerServer, &in_bytes); + if (result != 0) + { + return result; + } + + if (in_bytes == 0) + { + return 0; + } + else + { + logError("file: "__FILE__", line: %d, " \ + "server ip: %s, expect body length 0, " \ + "but received: "INT64_PRINTF_FORMAT, __LINE__, \ + pTrackerServer->ip_addr, in_bytes); + return EINVAL; + } +} + +int fdfs_deal_no_body_cmd_ex(const char *ip_addr, const int port, const int cmd) +{ + ConnectionInfo *conn; + ConnectionInfo server_info; + int result; + + strcpy(server_info.ip_addr, ip_addr); + server_info.port = port; + server_info.sock = -1; + if ((conn=tracker_connect_server(&server_info, &result)) == NULL) + { + return result; + } + + result = fdfs_deal_no_body_cmd(conn, cmd); + tracker_disconnect_server_ex(conn, result != 0); + return result; +} + +int fdfs_validate_group_name(const char *group_name) +{ + const char *p; + const char *pEnd; + int len; + + len = strlen(group_name); + if (len == 0) + { + return EINVAL; + } + + pEnd = group_name + len; + for (p=group_name; p= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || \ + (*p >= '0' && *p <= '9'))) + { + return EINVAL; + } + } + + return 0; +} + +int fdfs_validate_filename(const char *filename) +{ + const char *p; + const char *pEnd; + int len; + + len = strlen(filename); + pEnd = filename + len; + for (p=filename; p= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || \ + (*p >= '0' && *p <= '9') || (*p == '-') || (*p == '_')\ + || (*p == '.'))) + { + return EINVAL; + } + } + + return 0; +} + +int metadata_cmp_by_name(const void *p1, const void *p2) +{ + return strcmp(((FDFSMetaData *)p1)->name, ((FDFSMetaData *)p2)->name); +} + +const char *get_storage_status_caption(const int status) +{ + switch (status) + { + case FDFS_STORAGE_STATUS_INIT: + return "INIT"; + case FDFS_STORAGE_STATUS_WAIT_SYNC: + return "WAIT_SYNC"; + case FDFS_STORAGE_STATUS_SYNCING: + return "SYNCING"; + case FDFS_STORAGE_STATUS_OFFLINE: + return "OFFLINE"; + case FDFS_STORAGE_STATUS_ONLINE: + return "ONLINE"; + case FDFS_STORAGE_STATUS_DELETED: + return "DELETED"; + case FDFS_STORAGE_STATUS_IP_CHANGED: + return "IP_CHANGED"; + case FDFS_STORAGE_STATUS_ACTIVE: + return "ACTIVE"; + case FDFS_STORAGE_STATUS_RECOVERY: + return "RECOVERY"; + default: + return "UNKOWN"; + } +} + +FDFSMetaData *fdfs_split_metadata_ex(char *meta_buff, \ + const char recordSeperator, const char filedSeperator, \ + int *meta_count, int *err_no) +{ + char **rows; + char **ppRow; + char **ppEnd; + FDFSMetaData *meta_list; + FDFSMetaData *pMetadata; + char *pSeperator; + int nNameLen; + int nValueLen; + + *meta_count = getOccurCount(meta_buff, recordSeperator) + 1; + meta_list = (FDFSMetaData *)malloc(sizeof(FDFSMetaData) * \ + (*meta_count)); + if (meta_list == NULL) + { + *meta_count = 0; + *err_no = ENOMEM; + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(FDFSMetaData) * (*meta_count)); + return NULL; + } + + rows = (char **)malloc(sizeof(char *) * (*meta_count)); + if (rows == NULL) + { + free(meta_list); + *meta_count = 0; + *err_no = ENOMEM; + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", \ + __LINE__, (int)sizeof(char *) * (*meta_count)); + return NULL; + } + + *meta_count = splitEx(meta_buff, recordSeperator, \ + rows, *meta_count); + ppEnd = rows + (*meta_count); + pMetadata = meta_list; + for (ppRow=rows; ppRow FDFS_MAX_META_NAME_LEN) + { + nNameLen = FDFS_MAX_META_NAME_LEN; + } + if (nValueLen > FDFS_MAX_META_VALUE_LEN) + { + nValueLen = FDFS_MAX_META_VALUE_LEN; + } + + memcpy(pMetadata->name, *ppRow, nNameLen); + memcpy(pMetadata->value, pSeperator+1, nValueLen); + pMetadata->name[nNameLen] = '\0'; + pMetadata->value[nValueLen] = '\0'; + + pMetadata++; + } + + *meta_count = pMetadata - meta_list; + free(rows); + + *err_no = 0; + return meta_list; +} + +char *fdfs_pack_metadata(const FDFSMetaData *meta_list, const int meta_count, \ + char *meta_buff, int *buff_bytes) +{ + const FDFSMetaData *pMetaCurr; + const FDFSMetaData *pMetaEnd; + char *p; + int name_len; + int value_len; + + if (meta_buff == NULL) + { + meta_buff = (char *)malloc(sizeof(FDFSMetaData) * meta_count); + if (meta_buff == NULL) + { + *buff_bytes = 0; + + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail", __LINE__, \ + (int)sizeof(FDFSMetaData) * meta_count); + return NULL; + } + } + + p = meta_buff; + pMetaEnd = meta_list + meta_count; + for (pMetaCurr=meta_list; pMetaCurrname); + value_len = strlen(pMetaCurr->value); + memcpy(p, pMetaCurr->name, name_len); + p += name_len; + *p++ = FDFS_FIELD_SEPERATOR; + memcpy(p, pMetaCurr->value, value_len); + p += value_len; + *p++ = FDFS_RECORD_SEPERATOR; + } + + *(--p) = '\0'; //omit the last record seperator + *buff_bytes = p - meta_buff; + return meta_buff; +} + +void tracker_disconnect_server_ex(ConnectionInfo *conn, \ + const bool bForceClose) +{ + if (g_use_connection_pool) + { + conn_pool_close_connection_ex(&g_connection_pool, \ + conn, bForceClose); + } + else + { + conn_pool_disconnect_server(conn); + } +} + +ConnectionInfo *tracker_connect_server_ex(ConnectionInfo *pTrackerServer, \ + const int connect_timeout, int *err_no) +{ + if (g_use_connection_pool) + { + return conn_pool_get_connection(&g_connection_pool, + pTrackerServer, err_no); + } + else + { + *err_no = conn_pool_connect_server(pTrackerServer, \ + connect_timeout); + if (*err_no != 0) + { + return NULL; + } + else + { + return pTrackerServer; + } + } +} + +int tracker_connect_server_no_pool(ConnectionInfo *pTrackerServer) +{ + if (pTrackerServer->sock >= 0) + { + return 0; + } + + return conn_pool_connect_server(pTrackerServer, \ + g_fdfs_connect_timeout); +} + +static int fdfs_do_parameter_req(ConnectionInfo *pTrackerServer, \ + char *buff, const int buff_size) +{ + char out_buff[sizeof(TrackerHeader)]; + TrackerHeader *pHeader; + int64_t in_bytes; + int result; + + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ; + if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \ + sizeof(TrackerHeader), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d, send data fail, " \ + "errno: %d, error info: %s.", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + return result; + } + + result = fdfs_recv_response(pTrackerServer, &buff, buff_size, &in_bytes); + if (result != 0) + { + return result; + } + + if (in_bytes >= buff_size) + { + logError("file: "__FILE__", line: %d, " \ + "server: %s:%d, recv body bytes: " \ + INT64_PRINTF_FORMAT" exceed max: %d", \ + __LINE__, pTrackerServer->ip_addr, \ + pTrackerServer->port, in_bytes, buff_size); + return ENOSPC; + } + + *(buff + in_bytes) = '\0'; + return 0; +} + +int fdfs_get_ini_context_from_tracker(TrackerServerGroup *pTrackerGroup, \ + IniContext *iniContext, bool * volatile continue_flag, \ + const bool client_bind_addr, const char *bind_addr) +{ + ConnectionInfo *pGlobalServer; + ConnectionInfo *pTServer; + ConnectionInfo *pServerStart; + ConnectionInfo *pServerEnd; + ConnectionInfo trackerServer; + char in_buff[1024]; + int result; + int leader_index; + int i; + + result = 0; + pTServer = &trackerServer; + pServerEnd = pTrackerGroup->servers + pTrackerGroup->server_count; + + leader_index = pTrackerGroup->leader_index; + if (leader_index >= 0) + { + pServerStart = pTrackerGroup->servers + leader_index; + } + else + { + pServerStart = pTrackerGroup->servers; + } + + do + { + for (pGlobalServer=pServerStart; pGlobalServersock = socket(AF_INET, SOCK_STREAM, 0); + if(pTServer->sock < 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "socket create failed, errno: %d, " \ + "error info: %s.", \ + __LINE__, result, STRERROR(result)); + sleep(5); + break; + } + + if (client_bind_addr && (bind_addr != NULL && \ + *bind_addr != '\0')) + { + socketBind(pTServer->sock, bind_addr, 0); + } + + if (tcpsetnonblockopt(pTServer->sock) != 0) + { + close(pTServer->sock); + pTServer->sock = -1; + sleep(1); + continue; + } + + if ((result=connectserverbyip_nb(pTServer->sock, \ + pTServer->ip_addr, pTServer->port, \ + g_fdfs_connect_timeout)) == 0) + { + break; + } + + close(pTServer->sock); + pTServer->sock = -1; + sleep(1); + } + + if (pTServer->sock < 0) + { + logError("file: "__FILE__", line: %d, " \ + "connect to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTServer->ip_addr, pTServer->port, \ + result, STRERROR(result)); + + continue; + } + + result = fdfs_do_parameter_req(pTServer, in_buff, \ + sizeof(in_buff)); + if (result == 0) + { + result = iniLoadFromBuffer(in_buff, iniContext); + + close(pTServer->sock); + return result; + } + + fdfs_quit(pTServer); + close(pTServer->sock); + sleep(1); + } + + if (pServerStart != pTrackerGroup->servers) + { + pServerStart = pTrackerGroup->servers; + } + } while (*continue_flag); + + return EINTR; +} + diff --git a/tracker/tracker_proto.h b/tracker/tracker_proto.h new file mode 100644 index 0000000..596c3f9 --- /dev/null +++ b/tracker/tracker_proto.h @@ -0,0 +1,282 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_proto.h + +#ifndef _TRACKER_PROTO_H_ +#define _TRACKER_PROTO_H_ + +#include "tracker_types.h" +#include "connection_pool.h" +#include "ini_file_reader.h" + +#define TRACKER_PROTO_CMD_STORAGE_JOIN 81 +#define FDFS_PROTO_CMD_QUIT 82 +#define TRACKER_PROTO_CMD_STORAGE_BEAT 83 //storage heart beat +#define TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE 84 //report disk usage +#define TRACKER_PROTO_CMD_STORAGE_REPLICA_CHG 85 //repl new storage servers +#define TRACKER_PROTO_CMD_STORAGE_SYNC_SRC_REQ 86 //src storage require sync +#define TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ 87 //dest storage require sync +#define TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY 88 //sync done notify +#define TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT 89 //report src last synced time as dest server +#define TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_QUERY 79 //dest storage query sync src storage server +#define TRACKER_PROTO_CMD_STORAGE_REPORT_IP_CHANGED 78 //storage server report it's ip changed +#define TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ 77 //storage server request storage server's changelog +#define TRACKER_PROTO_CMD_STORAGE_REPORT_STATUS 76 //report specified storage server status +#define TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ 75 //storage server request parameters +#define TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FREE 74 //storage report trunk free space +#define TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID 73 //storage report current trunk file id +#define TRACKER_PROTO_CMD_STORAGE_FETCH_TRUNK_FID 72 //storage get current trunk file id +#define TRACKER_PROTO_CMD_STORAGE_GET_STATUS 71 //get storage status from tracker +#define TRACKER_PROTO_CMD_STORAGE_GET_SERVER_ID 70 //get storage server id from tracker +#define TRACKER_PROTO_CMD_STORAGE_FETCH_STORAGE_IDS 69 //get all storage ids from tracker + +#define TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_START 61 //start of tracker get system data files +#define TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_END 62 //end of tracker get system data files +#define TRACKER_PROTO_CMD_TRACKER_GET_ONE_SYS_FILE 63 //tracker get a system data file +#define TRACKER_PROTO_CMD_TRACKER_GET_STATUS 64 //tracker get status of other tracker +#define TRACKER_PROTO_CMD_TRACKER_PING_LEADER 65 //tracker ping leader +#define TRACKER_PROTO_CMD_TRACKER_NOTIFY_NEXT_LEADER 66 //notify next leader to other trackers +#define TRACKER_PROTO_CMD_TRACKER_COMMIT_NEXT_LEADER 67 //commit next leader to other trackers + +#define TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP 90 +#define TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS 91 +#define TRACKER_PROTO_CMD_SERVER_LIST_STORAGE 92 +#define TRACKER_PROTO_CMD_SERVER_DELETE_STORAGE 93 +#define TRACKER_PROTO_CMD_SERVER_SET_TRUNK_SERVER 94 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE 101 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE 102 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE 103 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE 104 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL 105 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL 106 +#define TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL 107 +#define TRACKER_PROTO_CMD_RESP 100 +#define FDFS_PROTO_CMD_ACTIVE_TEST 111 //active test, tracker and storage both support since V1.28 + +#define STORAGE_PROTO_CMD_REPORT_SERVER_ID 9 +#define STORAGE_PROTO_CMD_UPLOAD_FILE 11 +#define STORAGE_PROTO_CMD_DELETE_FILE 12 +#define STORAGE_PROTO_CMD_SET_METADATA 13 +#define STORAGE_PROTO_CMD_DOWNLOAD_FILE 14 +#define STORAGE_PROTO_CMD_GET_METADATA 15 +#define STORAGE_PROTO_CMD_SYNC_CREATE_FILE 16 +#define STORAGE_PROTO_CMD_SYNC_DELETE_FILE 17 +#define STORAGE_PROTO_CMD_SYNC_UPDATE_FILE 18 +#define STORAGE_PROTO_CMD_SYNC_CREATE_LINK 19 +#define STORAGE_PROTO_CMD_CREATE_LINK 20 +#define STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE 21 +#define STORAGE_PROTO_CMD_QUERY_FILE_INFO 22 +#define STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE 23 //create appender file +#define STORAGE_PROTO_CMD_APPEND_FILE 24 //append file +#define STORAGE_PROTO_CMD_SYNC_APPEND_FILE 25 +#define STORAGE_PROTO_CMD_FETCH_ONE_PATH_BINLOG 26 //fetch binlog of one store path +#define STORAGE_PROTO_CMD_RESP TRACKER_PROTO_CMD_RESP +#define STORAGE_PROTO_CMD_UPLOAD_MASTER_FILE STORAGE_PROTO_CMD_UPLOAD_FILE + +#define STORAGE_PROTO_CMD_TRUNK_ALLOC_SPACE 27 //since V3.00, storage to trunk server +#define STORAGE_PROTO_CMD_TRUNK_ALLOC_CONFIRM 28 //since V3.00, storage to trunk server +#define STORAGE_PROTO_CMD_TRUNK_FREE_SPACE 29 //since V3.00, storage to trunk server +#define STORAGE_PROTO_CMD_TRUNK_SYNC_BINLOG 30 //since V3.00, trunk storage to storage +#define STORAGE_PROTO_CMD_TRUNK_GET_BINLOG_SIZE 31 //since V3.07, tracker to storage +#define STORAGE_PROTO_CMD_TRUNK_DELETE_BINLOG_MARKS 32 //since V3.07, tracker to storage +#define STORAGE_PROTO_CMD_TRUNK_TRUNCATE_BINLOG_FILE 33 //since V3.07, trunk storage to storage + +#define STORAGE_PROTO_CMD_MODIFY_FILE 34 //since V3.08 +#define STORAGE_PROTO_CMD_SYNC_MODIFY_FILE 35 //since V3.08 +#define STORAGE_PROTO_CMD_TRUNCATE_FILE 36 //since V3.08 +#define STORAGE_PROTO_CMD_SYNC_TRUNCATE_FILE 37 //since V3.08 + +//for overwrite all old metadata +#define STORAGE_SET_METADATA_FLAG_OVERWRITE 'O' +#define STORAGE_SET_METADATA_FLAG_OVERWRITE_STR "O" +//for replace, insert when the meta item not exist, otherwise update it +#define STORAGE_SET_METADATA_FLAG_MERGE 'M' +#define STORAGE_SET_METADATA_FLAG_MERGE_STR "M" + +#define FDFS_PROTO_PKG_LEN_SIZE 8 +#define FDFS_PROTO_CMD_SIZE 1 +#define FDFS_PROTO_IP_PORT_SIZE (IP_ADDRESS_SIZE + 6) + +#define TRACKER_QUERY_STORAGE_FETCH_BODY_LEN (FDFS_GROUP_NAME_MAX_LEN \ + + IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE) +#define TRACKER_QUERY_STORAGE_STORE_BODY_LEN (FDFS_GROUP_NAME_MAX_LEN \ + + IP_ADDRESS_SIZE - 1 + FDFS_PROTO_PKG_LEN_SIZE + 1) + +#define STORAGE_TRUNK_ALLOC_CONFIRM_REQ_BODY_LEN (FDFS_GROUP_NAME_MAX_LEN \ + + sizeof(FDFSTrunkInfoBuff)) + +typedef struct +{ + char pkg_len[FDFS_PROTO_PKG_LEN_SIZE]; //body length, not including header + char cmd; //command code + char status; //status code for response +} TrackerHeader; + +typedef struct +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN+1]; + char storage_port[FDFS_PROTO_PKG_LEN_SIZE]; + char storage_http_port[FDFS_PROTO_PKG_LEN_SIZE]; + char store_path_count[FDFS_PROTO_PKG_LEN_SIZE]; + char subdir_count_per_path[FDFS_PROTO_PKG_LEN_SIZE]; + char upload_priority[FDFS_PROTO_PKG_LEN_SIZE]; + char join_time[FDFS_PROTO_PKG_LEN_SIZE]; //storage join timestamp + char up_time[FDFS_PROTO_PKG_LEN_SIZE]; //storage service started timestamp + char version[FDFS_VERSION_SIZE]; //storage version + char domain_name[FDFS_DOMAIN_NAME_MAX_SIZE]; + char init_flag; + signed char status; + char tracker_count[FDFS_PROTO_PKG_LEN_SIZE]; //all tracker server count +} TrackerStorageJoinBody; + +typedef struct +{ + char src_id[FDFS_STORAGE_ID_MAX_SIZE]; //src storage id +} TrackerStorageJoinBodyResp; + +typedef struct +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char sz_total_mb[FDFS_PROTO_PKG_LEN_SIZE]; //total disk storage in MB + char sz_free_mb[FDFS_PROTO_PKG_LEN_SIZE]; //free disk storage in MB + char sz_trunk_free_mb[FDFS_PROTO_PKG_LEN_SIZE]; //trunk free space in MB + char sz_count[FDFS_PROTO_PKG_LEN_SIZE]; //server count + char sz_storage_port[FDFS_PROTO_PKG_LEN_SIZE]; + char sz_storage_http_port[FDFS_PROTO_PKG_LEN_SIZE]; + char sz_active_count[FDFS_PROTO_PKG_LEN_SIZE]; //active server count + char sz_current_write_server[FDFS_PROTO_PKG_LEN_SIZE]; + char sz_store_path_count[FDFS_PROTO_PKG_LEN_SIZE]; + char sz_subdir_count_per_path[FDFS_PROTO_PKG_LEN_SIZE]; + char sz_current_trunk_file_id[FDFS_PROTO_PKG_LEN_SIZE]; +} TrackerGroupStat; + +typedef struct +{ + char status; + char id[FDFS_STORAGE_ID_MAX_SIZE]; + char ip_addr[IP_ADDRESS_SIZE]; + char domain_name[FDFS_DOMAIN_NAME_MAX_SIZE]; + char src_id[FDFS_STORAGE_ID_MAX_SIZE]; //src storage id + char version[FDFS_VERSION_SIZE]; + char sz_join_time[8]; + char sz_up_time[8]; + char sz_total_mb[8]; + char sz_free_mb[8]; + char sz_upload_priority[8]; + char sz_store_path_count[8]; + char sz_subdir_count_per_path[8]; + char sz_current_write_path[8]; + char sz_storage_port[8]; + char sz_storage_http_port[8]; + FDFSStorageStatBuff stat_buff; + char if_trunk_server; +} TrackerStorageStat; + +typedef struct +{ + char src_id[FDFS_STORAGE_ID_MAX_SIZE]; //src storage id + char until_timestamp[FDFS_PROTO_PKG_LEN_SIZE]; +} TrackerStorageSyncReqBody; + +typedef struct +{ + char sz_total_mb[8]; + char sz_free_mb[8]; +} TrackerStatReportReqBody; + +typedef struct +{ + unsigned char store_path_index; + unsigned char sub_path_high; + unsigned char sub_path_low; + char id[4]; + char offset[4]; + char size[4]; +} FDFSTrunkInfoBuff; + +#ifdef __cplusplus +extern "C" { +#endif + +#define tracker_connect_server(pTrackerServer, err_no) \ + tracker_connect_server_ex(pTrackerServer, g_fdfs_connect_timeout, err_no) + +/** +* connect to the tracker server +* params: +* pTrackerServer: tracker server +* connect_timeout: connect timeout in seconds +* err_no: return the error no +* return: ConnectionInfo pointer for success, NULL for fail +**/ +ConnectionInfo *tracker_connect_server_ex(ConnectionInfo *pTrackerServer, \ + const int connect_timeout, int *err_no); + + +/** +* connect to the tracker server directly without connection pool +* params: +* pTrackerServer: tracker server +* return: 0 for success, none zero for fail +**/ +int tracker_connect_server_no_pool(ConnectionInfo *pTrackerServer); + +#define tracker_disconnect_server(pTrackerServer) \ + tracker_disconnect_server_ex(pTrackerServer, false) + +/** +* close all connections to tracker servers +* params: +* pTrackerServer: tracker server +* bForceClose: if force close the connection when use connection pool +* return: +**/ +void tracker_disconnect_server_ex(ConnectionInfo *pTrackerServer, \ + const bool bForceClose); + +int fdfs_validate_group_name(const char *group_name); +int fdfs_validate_filename(const char *filename); +int metadata_cmp_by_name(const void *p1, const void *p2); + +const char *get_storage_status_caption(const int status); + +int fdfs_recv_header(ConnectionInfo *pTrackerServer, int64_t *in_bytes); + +int fdfs_recv_response(ConnectionInfo *pTrackerServer, \ + char **buff, const int buff_size, \ + int64_t *in_bytes); +int fdfs_quit(ConnectionInfo *pTrackerServer); + +#define fdfs_active_test(pTrackerServer) \ + fdfs_deal_no_body_cmd(pTrackerServer, FDFS_PROTO_CMD_ACTIVE_TEST) + +int fdfs_deal_no_body_cmd(ConnectionInfo *pTrackerServer, const int cmd); + +int fdfs_deal_no_body_cmd_ex(const char *ip_addr, const int port, const int cmd); + +#define fdfs_split_metadata(meta_buff, meta_count, err_no) \ + fdfs_split_metadata_ex(meta_buff, FDFS_RECORD_SEPERATOR, \ + FDFS_FIELD_SEPERATOR, meta_count, err_no) + +char *fdfs_pack_metadata(const FDFSMetaData *meta_list, const int meta_count, \ + char *meta_buff, int *buff_bytes); +FDFSMetaData *fdfs_split_metadata_ex(char *meta_buff, \ + const char recordSeperator, const char filedSeperator, \ + int *meta_count, int *err_no); + +int fdfs_get_ini_context_from_tracker(TrackerServerGroup *pTrackerGroup, \ + IniContext *iniContext, bool * volatile continue_flag, \ + const bool client_bind_addr, const char *bind_addr); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/tracker/tracker_relationship.c b/tracker/tracker_relationship.c new file mode 100644 index 0000000..e12863d --- /dev/null +++ b/tracker/tracker_relationship.c @@ -0,0 +1,558 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "sockopt.h" +#include "fdfs_global.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "tracker_global.h" +#include "tracker_proto.h" +#include "tracker_mem.h" +#include "tracker_relationship.h" + +bool g_if_leader_self = false; //if I am leader + +static int fdfs_ping_leader(ConnectionInfo *pTrackerServer) +{ + TrackerHeader header; + int result; + int success_count; + int64_t in_bytes; + char in_buff[(FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) * \ + FDFS_MAX_GROUPS]; + char *pInBuff; + char *p; + char *pEnd; + FDFSGroupInfo *pGroup; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char trunk_server_id[FDFS_STORAGE_ID_MAX_SIZE]; + + memset(&header, 0, sizeof(header)); + header.cmd = TRACKER_PROTO_CMD_TRACKER_PING_LEADER; + result = tcpsenddata_nb(pTrackerServer->sock, &header, \ + sizeof(header), g_fdfs_network_timeout); + if(result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server ip: %s, send data fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTrackerServer->ip_addr, \ + result, STRERROR(result)); + return result; + } + + pInBuff = in_buff; + if ((result=fdfs_recv_response(pTrackerServer, &pInBuff, \ + sizeof(in_buff), &in_bytes)) != 0) + { + return result; + } + + if (in_bytes == 0) + { + return 0; + } + else if (in_bytes % (FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_STORAGE_ID_MAX_SIZE) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server ip: %s, invalid body length: " \ + INT64_PRINTF_FORMAT, __LINE__, \ + pTrackerServer->ip_addr, in_bytes); + return EINVAL; + } + + success_count = 0; + memset(group_name, 0, sizeof(group_name)); + memset(trunk_server_id, 0, sizeof(trunk_server_id)); + + pEnd = in_buff + in_bytes; + for (p=in_buff; pip_addr, group_name); + continue; + } + + if (*trunk_server_id == '\0') + { + *(pGroup->last_trunk_server_id) = '\0'; + pGroup->pTrunkServer = NULL; + success_count++; + continue; + } + + pGroup->pTrunkServer = tracker_mem_get_storage(pGroup, \ + trunk_server_id); + if (pGroup->pTrunkServer == NULL) + { + logWarning("file: "__FILE__", line: %d, " \ + "tracker server ip: %s, group: %s, " \ + "trunk server: %s not exists", \ + __LINE__, pTrackerServer->ip_addr, \ + group_name, trunk_server_id); + } + snprintf(pGroup->last_trunk_server_id, sizeof( \ + pGroup->last_trunk_server_id), "%s", trunk_server_id); + success_count++; + } + + if (success_count > 0) + { + tracker_save_groups(); + } + + return 0; +} + +static int relationship_cmp_tracker_status(const void *p1, const void *p2) +{ + TrackerRunningStatus *pStatus1; + TrackerRunningStatus *pStatus2; + ConnectionInfo *pTrackerServer1; + ConnectionInfo *pTrackerServer2; + int sub; + + pStatus1 = (TrackerRunningStatus *)p1; + pStatus2 = (TrackerRunningStatus *)p2; + sub = pStatus1->if_leader - pStatus2->if_leader; + if (sub != 0) + { + return sub; + } + + sub = pStatus1->running_time - pStatus2->running_time; + if (sub != 0) + { + return sub; + } + + sub = pStatus2->restart_interval - pStatus1->restart_interval; + if (sub != 0) + { + return sub; + } + + pTrackerServer1 = pStatus1->pTrackerServer; + pTrackerServer2 = pStatus2->pTrackerServer; + sub = strcmp(pTrackerServer1->ip_addr, pTrackerServer2->ip_addr); + if (sub != 0) + { + return sub; + } + + return pTrackerServer1->port - pTrackerServer2->port; +} + +static int relationship_get_tracker_leader(TrackerRunningStatus *pTrackerStatus) +{ + ConnectionInfo *pTrackerServer; + ConnectionInfo *pTrackerEnd; + TrackerRunningStatus *pStatus; + TrackerRunningStatus trackerStatus[FDFS_MAX_TRACKERS]; + int count; + int result; + int r; + int i; + + memset(pTrackerStatus, 0, sizeof(TrackerRunningStatus)); + pStatus = trackerStatus; + result = 0; + pTrackerEnd = g_tracker_servers.servers + g_tracker_servers.server_count; + for (pTrackerServer=g_tracker_servers.servers; \ + pTrackerServerpTrackerServer = pTrackerServer; + r = tracker_mem_get_status(pTrackerServer, pStatus); + if (r == 0) + { + pStatus++; + } + else if (r != ENOENT) + { + result = r; + } + } + + count = pStatus - trackerStatus; + if (count == 0) + { + return result == 0 ? ENOENT : result; + } + + qsort(trackerStatus, count, sizeof(TrackerRunningStatus), \ + relationship_cmp_tracker_status); + + for (i=0; iip_addr, \ + trackerStatus[i].pTrackerServer->port, \ + trackerStatus[i].if_leader, \ + trackerStatus[i].running_time, \ + trackerStatus[i].restart_interval); + } + + memcpy(pTrackerStatus, trackerStatus + (count - 1), \ + sizeof(TrackerRunningStatus)); + return 0; +} + +#define relationship_notify_next_leader(pTrackerServer, pLeader, bConnectFail) \ + do_notify_leader_changed(pTrackerServer, pLeader, \ + TRACKER_PROTO_CMD_TRACKER_NOTIFY_NEXT_LEADER, bConnectFail) + +#define relationship_commit_next_leader(pTrackerServer, pLeader, bConnectFail) \ + do_notify_leader_changed(pTrackerServer, pLeader, \ + TRACKER_PROTO_CMD_TRACKER_COMMIT_NEXT_LEADER, bConnectFail) + +static int do_notify_leader_changed(ConnectionInfo *pTrackerServer, \ + ConnectionInfo *pLeader, const char cmd, bool *bConnectFail) +{ + char out_buff[sizeof(TrackerHeader) + FDFS_PROTO_IP_PORT_SIZE]; + char in_buff[1]; + ConnectionInfo *conn; + TrackerHeader *pHeader; + char *pInBuff; + int64_t in_bytes; + int result; + + pTrackerServer->sock = -1; + if ((conn=tracker_connect_server(pTrackerServer, &result)) == NULL) + { + *bConnectFail = true; + return result; + } + *bConnectFail = false; + + do + { + memset(out_buff, 0, sizeof(out_buff)); + pHeader = (TrackerHeader *)out_buff; + pHeader->cmd = cmd; + sprintf(out_buff + sizeof(TrackerHeader), "%s:%d", \ + pLeader->ip_addr, pLeader->port); + long2buff(FDFS_PROTO_IP_PORT_SIZE, pHeader->pkg_len); + if ((result=tcpsenddata_nb(conn->sock, out_buff, \ + sizeof(out_buff), g_fdfs_network_timeout)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "send data to tracker server %s:%d fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTrackerServer->ip_addr, \ + pTrackerServer->port, \ + result, STRERROR(result)); + + result = (result == ENOENT ? EACCES : result); + break; + } + + pInBuff = in_buff; + result = fdfs_recv_response(conn, &pInBuff, \ + 0, &in_bytes); + if (result != 0) + { + break; + } + + if (in_bytes != 0) + { + logError("file: "__FILE__", line: %d, " \ + "tracker server %s:%d response data " \ + "length: "INT64_PRINTF_FORMAT" is invalid, " \ + "expect length: %d.", __LINE__, \ + pTrackerServer->ip_addr, pTrackerServer->port, \ + in_bytes, 0); + result = EINVAL; + break; + } + } while (0); + + if (pTrackerServer->port == g_server_port && \ + is_local_host_ip(pTrackerServer->ip_addr)) + { + tracker_disconnect_server_ex(conn, true); + } + else + { + tracker_disconnect_server_ex(conn, result != 0); + } + + return result; +} + +static int relationship_notify_leader_changed(ConnectionInfo *pLeader) +{ + ConnectionInfo *pTrackerServer; + ConnectionInfo *pTrackerEnd; + int result; + bool bConnectFail; + int success_count; + + result = ENOENT; + pTrackerEnd = g_tracker_servers.servers + g_tracker_servers.server_count; + success_count = 0; + for (pTrackerServer=g_tracker_servers.servers; \ + pTrackerServerport == g_server_port && \ + is_local_host_ip(trackerStatus.pTrackerServer->ip_addr)) + { + if ((result=relationship_notify_leader_changed( \ + trackerStatus.pTrackerServer)) != 0) + { + return result; + } + + logInfo("file: "__FILE__", line: %d, " \ + "I am the new tracker leader %s:%d", \ + __LINE__, trackerStatus.pTrackerServer->ip_addr, \ + trackerStatus.pTrackerServer->port); + + tracker_mem_find_trunk_servers(); + } + else + { + if (trackerStatus.if_leader) + { + g_tracker_servers.leader_index = \ + trackerStatus.pTrackerServer - \ + g_tracker_servers.servers; + if (g_tracker_servers.leader_index < 0 || \ + g_tracker_servers.leader_index >= \ + g_tracker_servers.server_count) + { + g_tracker_servers.leader_index = -1; + return EINVAL; + } + + logInfo("file: "__FILE__", line: %d, " \ + "the tracker leader %s:%d", __LINE__, \ + trackerStatus.pTrackerServer->ip_addr, \ + trackerStatus.pTrackerServer->port); + } + else + { + logDebug("file: "__FILE__", line: %d, " \ + "waiting for leader notify", __LINE__); + return ENOENT; + } + } + + return 0; +} + +static int relationship_ping_leader() +{ + int result; + int leader_index; + ConnectionInfo *pTrackerServer; + + if (g_if_leader_self) + { + return 0; //do not need ping myself + } + + leader_index = g_tracker_servers.leader_index; + if (leader_index < 0) + { + return EINVAL; + } + + pTrackerServer = g_tracker_servers.servers + leader_index; + if (pTrackerServer->sock < 0) + { + if ((result=conn_pool_connect_server(pTrackerServer, \ + g_fdfs_connect_timeout)) != 0) + { + return result; + } + } + + if ((result=fdfs_ping_leader(pTrackerServer)) != 0) + { + close(pTrackerServer->sock); + pTrackerServer->sock = -1; + } + + return result; +} + +static void *relationship_thread_entrance(void* arg) +{ +#define MAX_SLEEP_SECONDS 10 + + int fail_count; + int sleep_seconds; + + fail_count = 0; + while (g_continue_flag) + { + sleep_seconds = 1; + if (g_tracker_servers.servers != NULL) + { + if (g_tracker_servers.leader_index < 0) + { + if (relationship_select_leader() != 0) + { + sleep_seconds = 1 + (int)((double)rand() + * (double)MAX_SLEEP_SECONDS / RAND_MAX); + } + } + else + { + if (relationship_ping_leader() == 0) + { + fail_count = 0; + } + else + { + fail_count++; + if (fail_count >= 3) + { + g_tracker_servers.leader_index = -1; + } + } + } + } + + if (g_last_tracker_servers != NULL) + { + tracker_mem_file_lock(); + + free(g_last_tracker_servers); + g_last_tracker_servers = NULL; + + tracker_mem_file_unlock(); + } + + sleep(sleep_seconds); + } + + return NULL; +} + +int tracker_relationship_init() +{ + int result; + pthread_t tid; + pthread_attr_t thread_attr; + + if ((result=init_pthread_attr(&thread_attr, g_thread_stack_size)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_attr fail, program exit!", __LINE__); + return result; + } + + if ((result=pthread_create(&tid, &thread_attr, \ + relationship_thread_entrance, NULL)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "create thread failed, errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + pthread_attr_destroy(&thread_attr); + + return 0; +} + +int tracker_relationship_destroy() +{ + return 0; +} + diff --git a/tracker/tracker_relationship.h b/tracker/tracker_relationship.h new file mode 100644 index 0000000..69fd9f3 --- /dev/null +++ b/tracker/tracker_relationship.h @@ -0,0 +1,32 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_relationship.h + +#ifndef _TRACKER_RELATIONSHIP_H_ +#define _TRACKER_RELATIONSHIP_H_ + +#include +#include +#include "tracker_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern bool g_if_leader_self; //if I am leader + +int tracker_relationship_init(); +int tracker_relationship_destroy(); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/tracker/tracker_service.c b/tracker/tracker_service.c new file mode 100644 index 0000000..7c31426 --- /dev/null +++ b/tracker/tracker_service.c @@ -0,0 +1,3725 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_service.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "base64.h" +#include "logger.h" +#include "fdfs_global.h" +#include "sockopt.h" +#include "shared_func.h" +#include "pthread_func.h" +#include "sched_thread.h" +#include "tracker_types.h" +#include "tracker_global.h" +#include "tracker_mem.h" +#include "tracker_func.h" +#include "tracker_proto.h" +#include "tracker_nio.h" +#include "tracker_relationship.h" +#include "fdfs_shared_func.h" +#include "ioevent_loop.h" +#include "tracker_service.h" + +#define PKG_LEN_PRINTF_FORMAT "%d" + +static pthread_mutex_t tracker_thread_lock; +static pthread_mutex_t lb_thread_lock; + +int g_tracker_thread_count = 0; +struct nio_thread_data *g_thread_data = NULL; + +static int lock_by_client_count = 0; + +static void *work_thread_entrance(void* arg); +static void wait_for_work_threads_exit(); +static void tracker_find_max_free_space_group(); + +int tracker_service_init() +{ + int result; + struct nio_thread_data *pThreadData; + struct nio_thread_data *pDataEnd; + pthread_t tid; + pthread_attr_t thread_attr; + + if ((result=init_pthread_lock(&tracker_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_lock(&lb_thread_lock)) != 0) + { + return result; + } + + if ((result=init_pthread_attr(&thread_attr, g_thread_stack_size)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "init_pthread_attr fail, program exit!", __LINE__); + return result; + } + + if ((result=free_queue_init(g_max_connections, TRACKER_MAX_PACKAGE_SIZE,\ + TRACKER_MAX_PACKAGE_SIZE, sizeof(TrackerClientInfo))) != 0) + { + return result; + } + + g_thread_data = (struct nio_thread_data *)malloc(sizeof( \ + struct nio_thread_data) * g_work_threads); + if (g_thread_data == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "malloc %d bytes fail, errno: %d, error info: %s", \ + __LINE__, (int)sizeof(struct nio_thread_data) * \ + g_work_threads, errno, STRERROR(errno)); + return errno != 0 ? errno : ENOMEM; + } + + g_tracker_thread_count = 0; + pDataEnd = g_thread_data + g_work_threads; + for (pThreadData=g_thread_data; pThreadDataev_puller, + g_max_connections + 2, 1000, 0) != 0) + { + result = errno != 0 ? errno : ENOMEM; + logError("file: "__FILE__", line: %d, " \ + "ioevent_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + result = fast_timer_init(&pThreadData->timer, + 2 * g_fdfs_network_timeout, g_current_time); + if (result != 0) + { + logError("file: "__FILE__", line: %d, " \ + "fast_timer_init fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + return result; + } + + if (pipe(pThreadData->pipe_fds) != 0) + { + result = errno != 0 ? errno : EPERM; + logError("file: "__FILE__", line: %d, " \ + "call pipe fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + break; + } + +#if defined(OS_LINUX) + if ((result=fd_add_flags(pThreadData->pipe_fds[0], \ + O_NONBLOCK | O_NOATIME)) != 0) + { + break; + } +#else + if ((result=fd_add_flags(pThreadData->pipe_fds[0], \ + O_NONBLOCK)) != 0) + { + break; + } +#endif + + if ((result=pthread_create(&tid, &thread_attr, \ + work_thread_entrance, pThreadData)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "create thread failed, startup threads: %d, " \ + "errno: %d, error info: %s", \ + __LINE__, g_tracker_thread_count, \ + result, STRERROR(result)); + break; + } + else + { + if ((result=pthread_mutex_lock(&tracker_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + g_tracker_thread_count++; + if ((result=pthread_mutex_unlock(&tracker_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + } + } + + pthread_attr_destroy(&thread_attr); + + return 0; +} + +int tracker_terminate_threads() +{ + struct nio_thread_data *pThreadData; + struct nio_thread_data *pDataEnd; + int quit_sock; + + if (g_thread_data != NULL) + { + pDataEnd = g_thread_data + g_work_threads; + quit_sock = 0; + for (pThreadData=g_thread_data; pThreadDatapipe_fds[1], &quit_sock, \ + sizeof(quit_sock)) != sizeof(quit_sock)) + { + logError("file: "__FILE__", line: %d, " \ + "write to pipe fail, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + } + } + + return 0; +} + +static void wait_for_work_threads_exit() +{ + while (g_tracker_thread_count != 0) + { + sleep(1); + } +} + +int tracker_service_destroy() +{ + wait_for_work_threads_exit(); + pthread_mutex_destroy(&tracker_thread_lock); + pthread_mutex_destroy(&lb_thread_lock); + + return 0; +} + +static void *accept_thread_entrance(void* arg) +{ + int server_sock; + int incomesock; + struct sockaddr_in inaddr; + socklen_t sockaddr_len; + struct nio_thread_data *pThreadData; + + server_sock = (long)arg; + while (g_continue_flag) + { + sockaddr_len = sizeof(inaddr); + incomesock = accept(server_sock, (struct sockaddr*)&inaddr, &sockaddr_len); + if (incomesock < 0) //error + { + if (!(errno == EINTR || errno == EAGAIN)) + { + logError("file: "__FILE__", line: %d, " \ + "accept failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + + continue; + } + + pThreadData = g_thread_data + incomesock % g_work_threads; + if (write(pThreadData->pipe_fds[1], &incomesock, \ + sizeof(incomesock)) != sizeof(incomesock)) + { + close(incomesock); + logError("file: "__FILE__", line: %d, " \ + "call write failed, " \ + "errno: %d, error info: %s", \ + __LINE__, errno, STRERROR(errno)); + } + } + + return NULL; +} + +void tracker_accept_loop(int server_sock) +{ + if (g_accept_threads > 1) + { + pthread_t tid; + pthread_attr_t thread_attr; + int result; + int i; + + if ((result=init_pthread_attr(&thread_attr, g_thread_stack_size)) != 0) + { + logWarning("file: "__FILE__", line: %d, " \ + "init_pthread_attr fail!", __LINE__); + } + else + { + for (i=1; iev_puller); + + if ((result=pthread_mutex_lock(&tracker_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + g_tracker_thread_count--; + if ((result=pthread_mutex_unlock(&tracker_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + return NULL; +} + +/* +storage server list +*/ +static int tracker_check_and_sync(struct fast_task_info *pTask, \ + const int status) +{ + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppEnd; + FDFSStorageDetail *pServer; + FDFSStorageBrief *pDestServer; + TrackerClientInfo *pClientInfo; + char *pFlags; + char *p; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + if (status != 0 || pClientInfo->pGroup == NULL) + { + pTask->length = sizeof(TrackerHeader); + return status; + } + + p = pTask->data + sizeof(TrackerHeader); + pFlags = p++; + *pFlags = 0; + if (g_if_leader_self) + { + if (pClientInfo->chg_count.tracker_leader != g_tracker_leader_chg_count) + { + int leader_index; + + *pFlags |= FDFS_CHANGE_FLAG_TRACKER_LEADER; + + pDestServer = (FDFSStorageBrief *)p; + memset(p, 0, sizeof(FDFSStorageBrief)); + + leader_index = g_tracker_servers.leader_index; + if (leader_index >= 0) + { + ConnectionInfo *pTServer; + pTServer = g_tracker_servers.servers + leader_index; + snprintf(pDestServer->id, FDFS_STORAGE_ID_MAX_SIZE, \ + "%s", pTServer->ip_addr); + memcpy(pDestServer->ip_addr, pTServer->ip_addr, \ + IP_ADDRESS_SIZE); + int2buff(pTServer->port, pDestServer->port); + } + pDestServer++; + + pClientInfo->chg_count.tracker_leader = \ + g_tracker_leader_chg_count; + p = (char *)pDestServer; + } + + if (pClientInfo->pStorage->trunk_chg_count != \ + pClientInfo->pGroup->trunk_chg_count) + { + *pFlags |= FDFS_CHANGE_FLAG_TRUNK_SERVER; + + pDestServer = (FDFSStorageBrief *)p; + memset(p, 0, sizeof(FDFSStorageBrief)); + + pServer = pClientInfo->pGroup->pTrunkServer; + if (pServer != NULL) + { + pDestServer->status = pServer->status; + memcpy(pDestServer->id, pServer->id, \ + FDFS_STORAGE_ID_MAX_SIZE); + memcpy(pDestServer->ip_addr, pServer->ip_addr, \ + IP_ADDRESS_SIZE); + int2buff(pClientInfo->pGroup->storage_port, \ + pDestServer->port); + } + pDestServer++; + + pClientInfo->pStorage->trunk_chg_count = \ + pClientInfo->pGroup->trunk_chg_count; + p = (char *)pDestServer; + } + } + + if (pClientInfo->pStorage->chg_count != pClientInfo->pGroup->chg_count) + { + *pFlags |= FDFS_CHANGE_FLAG_GROUP_SERVER; + + pDestServer = (FDFSStorageBrief *)p; + ppEnd = pClientInfo->pGroup->sorted_servers + \ + pClientInfo->pGroup->count; + for (ppServer=pClientInfo->pGroup->sorted_servers; \ + ppServerstatus = (*ppServer)->status; + memcpy(pDestServer->id, (*ppServer)->id, \ + FDFS_STORAGE_ID_MAX_SIZE); + memcpy(pDestServer->ip_addr, (*ppServer)->ip_addr, \ + IP_ADDRESS_SIZE); + int2buff(pClientInfo->pGroup->storage_port, \ + pDestServer->port); + pDestServer++; + } + + pClientInfo->pStorage->chg_count = \ + pClientInfo->pGroup->chg_count; + p = (char *)pDestServer; + } + + pTask->length = p - pTask->data; + return status; +} + +static int tracker_changelog_response(struct fast_task_info *pTask, \ + FDFSStorageDetail *pStorage) +{ + char filename[MAX_PATH_SIZE]; + int64_t changelog_fsize; + int read_bytes; + int chg_len; + int result; + int fd; + + changelog_fsize = g_changelog_fsize; + chg_len = changelog_fsize - pStorage->changelog_offset; + if (chg_len < 0) + { + chg_len = 0; + } + + if (chg_len == 0) + { + pTask->length = sizeof(TrackerHeader); + return 0; + } + + if (chg_len > sizeof(TrackerHeader) + TRACKER_MAX_PACKAGE_SIZE) + { + chg_len = TRACKER_MAX_PACKAGE_SIZE - sizeof(TrackerHeader); + } + + snprintf(filename, sizeof(filename), "%s/data/%s", g_fdfs_base_path,\ + STORAGE_SERVERS_CHANGELOG_FILENAME); + fd = open(filename, O_RDONLY); + if (fd < 0) + { + result = errno != 0 ? errno : EACCES; + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, open changelog file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + filename, result, STRERROR(result)); + pTask->length = sizeof(TrackerHeader); + return result; + } + + if (pStorage->changelog_offset > 0 && \ + lseek(fd, pStorage->changelog_offset, SEEK_SET) < 0) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, lseek changelog file %s fail, "\ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + filename, result, STRERROR(result)); + close(fd); + pTask->length = sizeof(TrackerHeader); + return result; + } + + read_bytes = read(fd, pTask->data + sizeof(TrackerHeader), chg_len); + close(fd); + + if (read_bytes != chg_len) + { + result = errno != 0 ? errno : EIO; + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, read changelog file %s fail, "\ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, \ + filename, result, STRERROR(result)); + + close(fd); + pTask->length = sizeof(TrackerHeader); + return result; + } + + pStorage->changelog_offset += chg_len; + tracker_save_storages(); + + pTask->length = sizeof(TrackerHeader) + chg_len; + return 0; +} + +static int tracker_deal_changelog_req(struct fast_task_info *pTask) +{ + int result; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *storage_id; + FDFSGroupInfo *pGroup; + FDFSStorageDetail *pStorage; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + pStorage = NULL; + + do + { + if (pClientInfo->pGroup != NULL && pClientInfo->pStorage != NULL) + { //already logined + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length = %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 0); + + result = EINVAL; + break; + } + + pStorage = pClientInfo->pStorage; + result = 0; + } + else + { + if (pTask->length - sizeof(TrackerHeader) != \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length = %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN + \ + FDFS_STORAGE_ID_MAX_SIZE); + + result = EINVAL; + break; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + result = ENOENT; + break; + } + + storage_id = pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN; + *(storage_id + FDFS_STORAGE_ID_MAX_SIZE - 1) = '\0'; + pStorage = tracker_mem_get_storage(pGroup, storage_id); + if (pStorage == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, group_name: %s, " \ + "storage server: %s not exist", \ + __LINE__, pTask->client_ip, \ + group_name, storage_id); + result = ENOENT; + break; + } + + result = 0; + } + } while (0); + + if (result != 0) + { + pTask->length = sizeof(TrackerHeader); + return result; + } + + return tracker_changelog_response(pTask, pStorage); +} + +static int tracker_deal_get_trunk_fid(struct fast_task_info *pTask) +{ + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length = %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_FETCH_TRUNK_FID, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 0); + + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader) + sizeof(int); + int2buff(pClientInfo->pGroup->current_trunk_file_id, \ + pTask->data + sizeof(TrackerHeader)); + + return 0; +} + +static int tracker_deal_parameter_req(struct fast_task_info *pTask) +{ + char reserved_space_str[32]; + + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length = %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 0); + + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader) + \ + sprintf(pTask->data + sizeof(TrackerHeader), \ + "use_storage_id=%d\n" \ + "id_type_in_filename=%s\n" \ + "storage_ip_changed_auto_adjust=%d\n" \ + "storage_sync_file_max_delay=%d\n" \ + "store_path=%d\n" \ + "reserved_storage_space=%s\n" \ + "use_trunk_file=%d\n" \ + "slot_min_size=%d\n" \ + "slot_max_size=%d\n" \ + "trunk_file_size=%d\n" \ + "trunk_create_file_advance=%d\n" \ + "trunk_create_file_time_base=%02d:%02d\n" \ + "trunk_create_file_interval=%d\n" \ + "trunk_create_file_space_threshold="INT64_PRINTF_FORMAT"\n" \ + "trunk_init_check_occupying=%d\n" \ + "trunk_init_reload_from_binlog=%d\n" \ + "trunk_compress_binlog_min_interval=%d\n" \ + "store_slave_file_use_link=%d\n", \ + g_use_storage_id, g_id_type_in_filename == \ + FDFS_ID_TYPE_SERVER_ID ? "id" : "ip", \ + g_storage_ip_changed_auto_adjust, \ + g_storage_sync_file_max_delay, g_groups.store_path, \ + fdfs_storage_reserved_space_to_string( \ + &g_storage_reserved_space, reserved_space_str), \ + g_if_use_trunk_file, \ + g_slot_min_size, g_slot_max_size, \ + g_trunk_file_size, g_trunk_create_file_advance, \ + g_trunk_create_file_time_base.hour, \ + g_trunk_create_file_time_base.minute, \ + g_trunk_create_file_interval, \ + g_trunk_create_file_space_threshold, \ + g_trunk_init_check_occupying, \ + g_trunk_init_reload_from_binlog, \ + g_trunk_compress_binlog_min_interval, \ + g_store_slave_file_use_link); + + return 0; +} + +static int tracker_deal_storage_replica_chg(struct fast_task_info *pTask) +{ + int server_count; + FDFSStorageBrief *briefServers; + int nPkgLen; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if ((nPkgLen <= 0) || (nPkgLen % sizeof(FDFSStorageBrief) != 0)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size %d is not correct", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_REPLICA_CHG, \ + pTask->client_ip, nPkgLen); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + server_count = nPkgLen / sizeof(FDFSStorageBrief); + if (server_count > FDFS_MAX_SERVERS_EACH_GROUP) + { + logError("file: "__FILE__", line: %d, " \ + "client ip addr: %s, return storage count: %d" \ + " exceed max: %d", __LINE__, \ + pTask->client_ip, server_count, \ + FDFS_MAX_SERVERS_EACH_GROUP); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + briefServers = (FDFSStorageBrief *)(pTask->data + sizeof(TrackerHeader)); + return tracker_mem_sync_storages(((TrackerClientInfo *)pTask->arg)->pGroup, \ + briefServers, server_count); +} + +static int tracker_deal_report_trunk_fid(struct fast_task_info *pTask) +{ + int current_trunk_fid; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + if (pTask->length - sizeof(TrackerHeader) != sizeof(int)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + current_trunk_fid = buff2int(pTask->data + sizeof(TrackerHeader)); + if (current_trunk_fid < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid current trunk file id: %d", \ + __LINE__, pTask->client_ip, current_trunk_fid); + return EINVAL; + } + + if (pClientInfo->pStorage != pClientInfo->pGroup->pTrunkServer) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, i am not the trunk server", \ + __LINE__, pTask->client_ip); + return EINVAL; + } + + if (pClientInfo->pGroup->current_trunk_file_id < current_trunk_fid) + { + pClientInfo->pGroup->current_trunk_file_id = current_trunk_fid; + return tracker_save_groups(); + } + else + { + return 0; + } +} + +static int tracker_deal_report_trunk_free_space(struct fast_task_info *pTask) +{ + int64_t trunk_free_space; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + if (pTask->length - sizeof(TrackerHeader) != 8) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FREE, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + trunk_free_space = buff2long(pTask->data + sizeof(TrackerHeader)); + if (trunk_free_space < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid trunk free space: " \ + INT64_PRINTF_FORMAT, __LINE__, pTask->client_ip, \ + trunk_free_space); + return EINVAL; + } + + if (pClientInfo->pStorage != pClientInfo->pGroup->pTrunkServer) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, i am not the trunk server", \ + __LINE__, pTask->client_ip); + return EINVAL; + } + + pClientInfo->pGroup->trunk_free_mb = trunk_free_space; + tracker_find_max_free_space_group(); + return 0; +} + +static int tracker_deal_notify_next_leader(struct fast_task_info *pTask) +{ + char *pIpAndPort; + char *ipAndPort[2]; + ConnectionInfo leader; + int server_index; + + if (pTask->length - sizeof(TrackerHeader) != FDFS_PROTO_IP_PORT_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct, expect length: %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), FDFS_PROTO_IP_PORT_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + *(pTask->data + pTask->length) = '\0'; + pIpAndPort = pTask->data + sizeof(TrackerHeader); + if (splitEx(pIpAndPort, ':', ipAndPort, 2) != 2) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid ip and port: %s", \ + __LINE__, pTask->client_ip, pIpAndPort); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + strcpy(leader.ip_addr, ipAndPort[0]); + leader.port = atoi(ipAndPort[1]); + server_index = fdfs_get_tracker_leader_index_ex(&g_tracker_servers, \ + leader.ip_addr, leader.port); + if (server_index < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, leader %s:%d not exist", \ + __LINE__, pTask->client_ip, \ + leader.ip_addr, leader.port); + return ENOENT; + } + + if (g_if_leader_self && (leader.port != g_server_port || \ + !is_local_host_ip(leader.ip_addr))) + { + g_if_leader_self = false; + g_tracker_servers.leader_index = -1; + g_tracker_leader_chg_count++; + + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, two leader occur, " \ + "new leader is %s:%d", \ + __LINE__, pTask->client_ip, \ + leader.ip_addr, leader.port); + return EINVAL; + } + + g_next_leader_index = server_index; + return 0; +} + +static int tracker_deal_commit_next_leader(struct fast_task_info *pTask) +{ + char *pIpAndPort; + char *ipAndPort[2]; + ConnectionInfo leader; + int server_index; + + if (pTask->length - sizeof(TrackerHeader) != FDFS_PROTO_IP_PORT_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct, expect length: %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), FDFS_PROTO_IP_PORT_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + *(pTask->data + pTask->length) = '\0'; + pIpAndPort = pTask->data + sizeof(TrackerHeader); + if (splitEx(pIpAndPort, ':', ipAndPort, 2) != 2) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid ip and port: %s", \ + __LINE__, pTask->client_ip, pIpAndPort); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + strcpy(leader.ip_addr, ipAndPort[0]); + leader.port = atoi(ipAndPort[1]); + server_index = fdfs_get_tracker_leader_index_ex(&g_tracker_servers, \ + leader.ip_addr, leader.port); + if (server_index < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, leader %s:%d not exist", \ + __LINE__, pTask->client_ip, \ + leader.ip_addr, leader.port); + return ENOENT; + } + if (server_index != g_next_leader_index) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, can't commit leader %s:%d", \ + __LINE__, pTask->client_ip, \ + leader.ip_addr, leader.port); + return EINVAL; + } + + g_tracker_servers.leader_index = server_index; + if (leader.port == g_server_port && is_local_host_ip(leader.ip_addr)) + { + g_if_leader_self = true; + g_tracker_leader_chg_count++; + } + else + { + logInfo("file: "__FILE__", line: %d, " \ + "the tracker leader is %s:%d", __LINE__, \ + leader.ip_addr, leader.port); + } + + return 0; +} + + +static int tracker_deal_server_get_storage_status(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char ip_addr[IP_ADDRESS_SIZE]; + FDFSGroupInfo *pGroup; + FDFSStorageDetail *pStorage; + FDFSStorageBrief *pDest; + int nPkgLen; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen < FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size %d is not correct", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_GET_STATUS, \ + pTask->client_ip, nPkgLen); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (nPkgLen == FDFS_GROUP_NAME_MAX_LEN) + { + strcpy(ip_addr, pTask->client_ip); + } + else + { + int ip_len; + + ip_len = nPkgLen - FDFS_GROUP_NAME_MAX_LEN; + if (ip_len >= IP_ADDRESS_SIZE) + { + ip_len = IP_ADDRESS_SIZE - 1; + } + memcpy(ip_addr, pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, ip_len); + *(ip_addr + ip_len) = '\0'; + } + + pStorage = tracker_mem_get_storage_by_ip(pGroup, ip_addr); + if (pStorage == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, group_name: %s, ip_addr: %s, " \ + "storage server not exist", __LINE__, \ + pTask->client_ip, group_name, ip_addr); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pTask->length = sizeof(TrackerHeader) + sizeof(FDFSStorageBrief); + pDest = (FDFSStorageBrief *)(pTask->data + sizeof(TrackerHeader)); + memset(pDest, 0, sizeof(FDFSStorageBrief)); + strcpy(pDest->id, pStorage->id); + strcpy(pDest->ip_addr, pStorage->ip_addr); + pDest->status = pStorage->status; + int2buff(pGroup->storage_port, pDest->port); + + return 0; +} + +static int tracker_deal_get_storage_id(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char ip_addr[IP_ADDRESS_SIZE]; + FDFSStorageIdInfo *pFDFSStorageIdInfo; + char *storage_id; + int nPkgLen; + int id_len; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen < FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size %d is not correct", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_GET_SERVER_ID, \ + pTask->client_ip, nPkgLen); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + + if (nPkgLen == FDFS_GROUP_NAME_MAX_LEN) + { + strcpy(ip_addr, pTask->client_ip); + } + else + { + int ip_len; + + ip_len = nPkgLen - FDFS_GROUP_NAME_MAX_LEN; + if (ip_len >= IP_ADDRESS_SIZE) + { + ip_len = IP_ADDRESS_SIZE - 1; + } + memcpy(ip_addr, pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, ip_len); + *(ip_addr + ip_len) = '\0'; + } + + if (g_use_storage_id) + { + pFDFSStorageIdInfo = fdfs_get_storage_id_by_ip(group_name, \ + ip_addr); + if (pFDFSStorageIdInfo == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "group_name: %s, storage ip: %s not exist", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_GET_SERVER_ID, \ + pTask->client_ip, group_name, ip_addr); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + storage_id = pFDFSStorageIdInfo->id; + } + else + { + storage_id = ip_addr; + } + + id_len = strlen(storage_id); + pTask->length = sizeof(TrackerHeader) + id_len; + memcpy(pTask->data + sizeof(TrackerHeader), storage_id, id_len); + + return 0; +} + +static int tracker_deal_fetch_storage_ids(struct fast_task_info *pTask) +{ + FDFSStorageIdInfo *pIdsStart; + FDFSStorageIdInfo *pIdsEnd; + FDFSStorageIdInfo *pIdInfo; + char *p; + int *pCurrentCount; + int nPkgLen; + int start_index; + + if (!g_use_storage_id) + { + logError("file: "__FILE__", line: %d, " \ + "client ip addr: %s, operation not supported", \ + __LINE__, pTask->client_ip); + pTask->length = sizeof(TrackerHeader); + return EOPNOTSUPP; + } + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen != sizeof(int)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size %d is not correct, " \ + "expect %d bytes", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_FETCH_STORAGE_IDS, \ + pTask->client_ip, nPkgLen, (int)sizeof(int)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + start_index = buff2int(pTask->data + sizeof(TrackerHeader)); + if (start_index < 0 || start_index >= g_storage_id_count) + { + logError("file: "__FILE__", line: %d, " \ + "client ip addr: %s, invalid offset: %d", \ + __LINE__, pTask->client_ip, start_index); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + int2buff(g_storage_id_count, p); + p += sizeof(int); + pCurrentCount = (int *)p; + p += sizeof(int); + + pIdsStart = g_storage_ids_by_ip + start_index; + pIdsEnd = g_storage_ids_by_ip + g_storage_id_count; + for (pIdInfo = pIdsStart; pIdInfo < pIdsEnd; pIdInfo++) + { + if ((int)(p - pTask->data) > pTask->size - 64) + { + break; + } + + p += sprintf(p, "%s %s %s\n", pIdInfo->id, \ + pIdInfo->group_name, pIdInfo->ip_addr); + } + + int2buff((int)(pIdInfo - pIdsStart), (char *)pCurrentCount); + pTask->length = p - pTask->data; + + return 0; +} + +static int tracker_deal_storage_report_status(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSGroupInfo *pGroup; + FDFSStorageBrief *briefServers; + + if (pTask->length - sizeof(TrackerHeader) != FDFS_GROUP_NAME_MAX_LEN + \ + sizeof(FDFSStorageBrief)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip addr: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_STATUS, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pTask->length = sizeof(TrackerHeader); + briefServers = (FDFSStorageBrief *)(pTask->data + \ + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN); + return tracker_mem_sync_storages(pGroup, briefServers, 1); +} + +static int tracker_deal_storage_join(struct fast_task_info *pTask) +{ + TrackerStorageJoinBodyResp *pJoinBodyResp; + TrackerStorageJoinBody *pBody; + ConnectionInfo *pTrackerServer; + ConnectionInfo *pTrackerEnd; + char *p; + char *pSeperator; + FDFSStorageJoinBody joinBody; + int result; + TrackerClientInfo *pClientInfo; + char tracker_ip[IP_ADDRESS_SIZE]; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + if (pTask->length - sizeof(TrackerHeader) < \ + sizeof(TrackerStorageJoinBody)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd: %d, client ip: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct, expect length >= %d.", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_JOIN, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), + (int)sizeof(TrackerStorageJoinBody)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pBody = (TrackerStorageJoinBody *)(pTask->data + sizeof(TrackerHeader)); + joinBody.tracker_count = buff2long(pBody->tracker_count); + if (joinBody.tracker_count <= 0 || \ + joinBody.tracker_count > FDFS_MAX_TRACKERS) + { + logError("file: "__FILE__", line: %d, " \ + "cmd: %d, client ip: %s, " \ + "tracker_count: %d is invalid, it <= 0 or > %d", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_JOIN, \ + pTask->client_ip, joinBody.tracker_count, \ + FDFS_MAX_TRACKERS); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + if (pTask->length - sizeof(TrackerHeader) != \ + sizeof(TrackerStorageJoinBody) + joinBody.tracker_count *\ + FDFS_PROTO_IP_PORT_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd: %d, client ip: %s, " \ + "package size "PKG_LEN_PRINTF_FORMAT" " \ + "is not correct, expect length %d.", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_JOIN, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), + (int)sizeof(TrackerStorageJoinBody) + \ + joinBody.tracker_count * FDFS_PROTO_IP_PORT_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(joinBody.group_name, pBody->group_name, FDFS_GROUP_NAME_MAX_LEN); + joinBody.group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0'; + if ((result=fdfs_validate_group_name(joinBody.group_name)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, \ + joinBody.group_name); + pTask->length = sizeof(TrackerHeader); + return result; + } + + joinBody.storage_port = (int)buff2long(pBody->storage_port); + if (joinBody.storage_port <= 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid port: %d", \ + __LINE__, pTask->client_ip, \ + joinBody.storage_port); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + joinBody.storage_http_port = (int)buff2long(pBody->storage_http_port); + if (joinBody.storage_http_port < 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid http port: %d", \ + __LINE__, pTask->client_ip, \ + joinBody.storage_http_port); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + joinBody.store_path_count = (int)buff2long(pBody->store_path_count); + if (joinBody.store_path_count <= 0 || joinBody.store_path_count > 256) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid store_path_count: %d", \ + __LINE__, pTask->client_ip, \ + joinBody.store_path_count); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + joinBody.subdir_count_per_path = (int)buff2long( \ + pBody->subdir_count_per_path); + if (joinBody.subdir_count_per_path <= 0 || \ + joinBody.subdir_count_per_path > 256) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid subdir_count_per_path: %d", \ + __LINE__, pTask->client_ip, \ + joinBody.subdir_count_per_path); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data+sizeof(TrackerHeader)+sizeof(TrackerStorageJoinBody); + pTrackerEnd = joinBody.tracker_servers + \ + joinBody.tracker_count; + for (pTrackerServer=joinBody.tracker_servers; \ + pTrackerServerclient_ip, p); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + *pSeperator = '\0'; + snprintf(pTrackerServer->ip_addr, \ + sizeof(pTrackerServer->ip_addr), "%s", p); + pTrackerServer->port = atoi(pSeperator + 1); + pTrackerServer->sock = -1; + + p += FDFS_PROTO_IP_PORT_SIZE; + } + + joinBody.upload_priority = (int)buff2long(pBody->upload_priority); + joinBody.join_time = (time_t)buff2long(pBody->join_time); + joinBody.up_time = (time_t)buff2long(pBody->up_time); + + *(pBody->version + (sizeof(pBody->version) - 1)) = '\0'; + *(pBody->domain_name + (sizeof(pBody->domain_name) - 1)) = '\0'; + strcpy(joinBody.version, pBody->version); + strcpy(joinBody.domain_name, pBody->domain_name); + joinBody.init_flag = pBody->init_flag; + joinBody.status = pBody->status; + + getSockIpaddr(pTask->event.fd, \ + tracker_ip, IP_ADDRESS_SIZE); + insert_into_local_host_ip(tracker_ip); + + result = tracker_mem_add_group_and_storage(pClientInfo, \ + pTask->client_ip, &joinBody, true); + if (result != 0) + { + pTask->length = sizeof(TrackerHeader); + return result; + } + + pJoinBodyResp = (TrackerStorageJoinBodyResp *)(pTask->data + \ + sizeof(TrackerHeader)); + memset(pJoinBodyResp, 0, sizeof(TrackerStorageJoinBodyResp)); + + if (pClientInfo->pStorage->psync_src_server != NULL) + { + strcpy(pJoinBodyResp->src_id, \ + pClientInfo->pStorage->psync_src_server->id); + } + + pTask->length = sizeof(TrackerHeader) + \ + sizeof(TrackerStorageJoinBodyResp); + return 0; +} + +static int tracker_deal_server_delete_storage(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *pStorageId; + FDFSGroupInfo *pGroup; + int nPkgLen; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen <= FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_DELETE_STORAGE, \ + pTask->client_ip, nPkgLen, FDFS_GROUP_NAME_MAX_LEN); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + if (nPkgLen >= FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length < %d", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_DELETE_STORAGE, \ + pTask->client_ip, nPkgLen, \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->data[pTask->length] = '\0'; + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0'; + pStorageId = pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN; + *(pStorageId + FDFS_STORAGE_ID_MAX_SIZE - 1) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pTask->length = sizeof(TrackerHeader); + return tracker_mem_delete_storage(pGroup, pStorageId); +} + +static int tracker_deal_server_set_trunk_server(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *pStorageId; + FDFSGroupInfo *pGroup; + const FDFSStorageDetail *pTrunkServer; + int nPkgLen; + int result; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen < FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length >= %d", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_SET_TRUNK_SERVER, \ + pTask->client_ip, nPkgLen, FDFS_GROUP_NAME_MAX_LEN); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + if (nPkgLen >= FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length < %d", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_SET_TRUNK_SERVER, \ + pTask->client_ip, nPkgLen, \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->data[pTask->length] = '\0'; + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0'; + pStorageId = pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN; + *(pStorageId + FDFS_STORAGE_ID_MAX_SIZE - 1) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pTrunkServer = tracker_mem_set_trunk_server(pGroup, \ + pStorageId, &result); + if (result == 0 && pTrunkServer != NULL) + { + int nIdLen; + nIdLen = strlen(pTrunkServer->id) + 1; + pTask->length = sizeof(TrackerHeader) + nIdLen; + memcpy(pTask->data + sizeof(TrackerHeader), \ + pTrunkServer->id, nIdLen); + } + else + { + if (result == 0) + { + result = ENOENT; + } + + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, set trunk server %s:%s fail, " \ + "errno: %d, error info: %s", __LINE__, \ + pTask->client_ip, group_name, pStorageId, \ + result, STRERROR(result)); + pTask->length = sizeof(TrackerHeader); + } + return result; +} + +static int tracker_deal_active_test(struct fast_task_info *pTask) +{ + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length 0", __LINE__, \ + FDFS_PROTO_CMD_ACTIVE_TEST, pTask->client_ip, \ + pTask->length - (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + return 0; +} + +static int tracker_deal_ping_leader(struct fast_task_info *pTask) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + int body_len; + char *p; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length 0", __LINE__, \ + TRACKER_PROTO_CMD_TRACKER_PING_LEADER, \ + pTask->client_ip, \ + pTask->length - (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + if (!g_if_leader_self) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, i am not the leader!", \ + __LINE__, TRACKER_PROTO_CMD_TRACKER_PING_LEADER, \ + pTask->client_ip); + pTask->length = sizeof(TrackerHeader); + return EOPNOTSUPP; + } + + if (pClientInfo->chg_count.trunk_server == g_trunk_server_chg_count) + { + pTask->length = sizeof(TrackerHeader); + return 0; + } + + body_len = (FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) * \ + g_groups.count; + if (body_len + sizeof(TrackerHeader) > pTask->size) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, " \ + "exceeds max package size: %d!", \ + __LINE__, TRACKER_PROTO_CMD_TRACKER_PING_LEADER, \ + pTask->client_ip, pTask->size); + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + + p = pTask->data + sizeof(TrackerHeader); + memset(p, 0, body_len); + + ppEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; ppGroupgroup_name, FDFS_GROUP_NAME_MAX_LEN); + p += FDFS_GROUP_NAME_MAX_LEN; + + if ((*ppGroup)->pTrunkServer != NULL) + { + memcpy(p, (*ppGroup)->pTrunkServer->id, \ + FDFS_STORAGE_ID_MAX_SIZE); + } + p += FDFS_STORAGE_ID_MAX_SIZE; + } + + pTask->length = p - pTask->data; + pClientInfo->chg_count.trunk_server = g_trunk_server_chg_count; + + return 0; +} + +static int tracker_unlock_by_client(struct fast_task_info *pTask) +{ + if (lock_by_client_count <= 0 || pTask->finish_callback == NULL) + { //already unlocked + return 0; + } + + pTask->finish_callback = NULL; + lock_by_client_count--; + + tracker_mem_file_unlock(); + + logInfo("file: "__FILE__", line: %d, " \ + "unlock by client: %s, locked count: %d", \ + __LINE__, pTask->client_ip, lock_by_client_count); + + return 0; +} + +static int tracker_lock_by_client(struct fast_task_info *pTask) +{ + if (lock_by_client_count > 0) + { + return EBUSY; + } + + tracker_mem_file_lock(); //avoid to read dirty data + + pTask->finish_callback = tracker_unlock_by_client; //make sure to release lock + lock_by_client_count++; + + logInfo("file: "__FILE__", line: %d, " \ + "lock by client: %s, locked count: %d", \ + __LINE__, pTask->client_ip, lock_by_client_count); + + return 0; +} + +/** +request package format: + none +response package format: + FDFS_PROTO_PKG_LEN_SIZE bytes: running time + FDFS_PROTO_PKG_LEN_SIZE bytes: startup interval + 1 byte: if leader +*/ +static int tracker_deal_get_tracker_status(struct fast_task_info *pTask) +{ + char *p; + TrackerRunningStatus runningStatus; + + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length %d", __LINE__, \ + TRACKER_PROTO_CMD_TRACKER_GET_STATUS, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 0); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + if (g_groups.count <= 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + p = pTask->data + sizeof(TrackerHeader); + + tracker_calc_running_times(&runningStatus); + + *p++= g_if_leader_self; //if leader + + long2buff(runningStatus.running_time, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + long2buff(runningStatus.restart_interval, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + pTask->length = p - pTask->data; + return 0; +} + +static int tracker_deal_get_sys_files_start(struct fast_task_info *pTask) +{ + int result; + + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length %d", __LINE__, \ + TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_START, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 0); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + if (g_groups.count == 0) + { + return ENOENT; + } + + if ((result=tracker_save_sys_files()) != 0) + { + return result == ENOENT ? EACCES: result; + } + + return tracker_lock_by_client(pTask); +} + +static int tracker_deal_get_sys_files_end(struct fast_task_info *pTask) +{ + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length %d", __LINE__, \ + TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_END, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 0); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + return tracker_unlock_by_client(pTask); +} + +/** +request package format: + 1 byte: filename index based 0 + FDFS_PROTO_PKG_LEN_SIZE bytes: offset +response package format: + FDFS_PROTO_PKG_LEN_SIZE bytes: file size + file size: file content +*/ +static int tracker_deal_get_one_sys_file(struct fast_task_info *pTask) +{ +#define TRACKER_READ_BYTES_ONCE (TRACKER_MAX_PACKAGE_SIZE - \ + FDFS_PROTO_PKG_LEN_SIZE - sizeof(TrackerHeader) - 1) + int result; + int index; + struct stat file_stat; + int64_t offset; + int64_t read_bytes; + int64_t bytes; + char full_filename[MAX_PATH_SIZE]; + char *p; + + if (pTask->length - sizeof(TrackerHeader) != 1+FDFS_PROTO_PKG_LEN_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length %d", __LINE__, \ + TRACKER_PROTO_CMD_TRACKER_GET_ONE_SYS_FILE, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), 1+FDFS_PROTO_PKG_LEN_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + p = pTask->data + sizeof(TrackerHeader); + index = *p++; + offset = buff2long(p); + + if (index < 0 || index >= TRACKER_SYS_FILE_COUNT) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid index: %d", \ + __LINE__, pTask->client_ip, index); + + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + g_fdfs_base_path, g_tracker_sys_filenames[index]); + if (stat(full_filename, &file_stat) != 0) + { + result = errno != 0 ? errno : ENOENT; + logError("file: "__FILE__", line: %d, " \ + "client ip:%s, call stat file %s fail, " \ + "errno: %d, error info: %s", \ + __LINE__, pTask->client_ip, full_filename, + result, STRERROR(result)); + return result; + } + + if (offset < 0 || offset > file_stat.st_size) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid offset: "INT64_PRINTF_FORMAT + " < 0 or > "OFF_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, offset, \ + file_stat.st_size); + + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + read_bytes = file_stat.st_size - offset; + if (read_bytes > TRACKER_READ_BYTES_ONCE) + { + read_bytes = TRACKER_READ_BYTES_ONCE; + } + + p = pTask->data + sizeof(TrackerHeader); + long2buff(file_stat.st_size, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + bytes = read_bytes; + if (read_bytes > 0 && (result=getFileContentEx(full_filename, \ + p, offset, &bytes)) != 0) + { + pTask->length = sizeof(TrackerHeader); + return result; + } + + if (bytes != read_bytes) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, read bytes: "INT64_PRINTF_FORMAT + " != expect bytes: "INT64_PRINTF_FORMAT, \ + __LINE__, pTask->client_ip, bytes, read_bytes); + + pTask->length = sizeof(TrackerHeader); + return EIO; + } + + p += read_bytes; + pTask->length = p - pTask->data; + return 0; +} + +static int tracker_deal_storage_report_ip_changed(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSGroupInfo *pGroup; + char *pOldIpAddr; + char *pNewIpAddr; + + if (g_use_storage_id) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, do NOT support ip changed adjust " \ + "because cluster use server ID instead of " \ + "IP address", __LINE__, pTask->client_ip); + return EOPNOTSUPP; + } + + if (pTask->length - sizeof(TrackerHeader) != \ + FDFS_GROUP_NAME_MAX_LEN + 2 * IP_ADDRESS_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length = %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_IP_CHANGED, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader),\ + FDFS_GROUP_NAME_MAX_LEN + 2 * IP_ADDRESS_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + + pOldIpAddr = pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN; + *(pOldIpAddr + (IP_ADDRESS_SIZE - 1)) = '\0'; + + pNewIpAddr = pOldIpAddr + IP_ADDRESS_SIZE; + *(pNewIpAddr + (IP_ADDRESS_SIZE - 1)) = '\0'; + + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (strcmp(pNewIpAddr, pTask->client_ip) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, group_name: %s, " \ + "new ip address %s != client ip address %s", \ + __LINE__, pTask->client_ip, group_name, \ + pNewIpAddr, pTask->client_ip); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + return tracker_mem_storage_ip_changed(pGroup, \ + pOldIpAddr, pNewIpAddr); +} + +static int tracker_deal_storage_sync_notify(struct fast_task_info *pTask) +{ + TrackerStorageSyncReqBody *pBody; + char sync_src_id[FDFS_STORAGE_ID_MAX_SIZE]; + bool bSaveStorages; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + if (pTask->length - sizeof(TrackerHeader) != \ + sizeof(TrackerStorageSyncReqBody)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd: %d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), + (int)sizeof(TrackerStorageSyncReqBody)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pBody=(TrackerStorageSyncReqBody *)(pTask->data+sizeof(TrackerHeader)); + if (*(pBody->src_id) == '\0') + { + if (pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_INIT || \ + pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_WAIT_SYNC || \ + pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_SYNCING) + { + pClientInfo->pStorage->status = FDFS_STORAGE_STATUS_ONLINE; + pClientInfo->pGroup->chg_count++; + tracker_save_storages(); + } + + pTask->length = sizeof(TrackerHeader); + return 0; + } + + bSaveStorages = false; + if (pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_INIT) + { + pClientInfo->pStorage->status = FDFS_STORAGE_STATUS_WAIT_SYNC; + pClientInfo->pGroup->chg_count++; + bSaveStorages = true; + } + + if (pClientInfo->pStorage->psync_src_server == NULL) + { + memcpy(sync_src_id, pBody->src_id, \ + FDFS_STORAGE_ID_MAX_SIZE); + sync_src_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0'; + + pClientInfo->pStorage->psync_src_server = \ + tracker_mem_get_storage(pClientInfo->pGroup, \ + sync_src_id); + if (pClientInfo->pStorage->psync_src_server == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, " \ + "sync src server: %s not exists", \ + __LINE__, pTask->client_ip, \ + sync_src_id); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (pClientInfo->pStorage->psync_src_server->status == \ + FDFS_STORAGE_STATUS_DELETED) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, " \ + "sync src server: %s already be deleted", \ + __LINE__, pTask->client_ip, \ + sync_src_id); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (pClientInfo->pStorage->psync_src_server->status == \ + FDFS_STORAGE_STATUS_IP_CHANGED) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, the ip address of " \ + "the sync src server: %s changed", \ + __LINE__, pTask->client_ip, \ + sync_src_id); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pClientInfo->pStorage->sync_until_timestamp = \ + (int)buff2long(pBody->until_timestamp); + bSaveStorages = true; + } + + if (bSaveStorages) + { + tracker_save_storages(); + } + + pTask->length = sizeof(TrackerHeader); + return 0; +} + +/** +pkg format: +Header +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +**/ +static int tracker_deal_server_list_group_storages(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char storage_id[FDFS_STORAGE_ID_MAX_SIZE]; + char *pStorageId; + FDFSGroupInfo *pGroup; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppEnd; + FDFSStorageStat *pStorageStat; + TrackerStorageStat *pStart; + TrackerStorageStat *pDest; + FDFSStorageStatBuff *pStatBuff; + int nPkgLen; + int id_len; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen < FDFS_GROUP_NAME_MAX_LEN || \ + nPkgLen >= FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length >= %d && <= %d", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_LIST_STORAGE, \ + pTask->client_ip, \ + nPkgLen, FDFS_GROUP_NAME_MAX_LEN, \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (nPkgLen > FDFS_GROUP_NAME_MAX_LEN) + { + id_len = nPkgLen - FDFS_GROUP_NAME_MAX_LEN; + if (id_len >= sizeof(storage_id)) + { + id_len = sizeof(storage_id) - 1; + } + pStorageId = storage_id; + memcpy(pStorageId, pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN, id_len); + *(pStorageId + id_len) = '\0'; + } + else + { + pStorageId = NULL; + } + + memset(pTask->data + sizeof(TrackerHeader), 0, \ + pTask->size - sizeof(TrackerHeader)); + pDest = pStart = (TrackerStorageStat *)(pTask->data + \ + sizeof(TrackerHeader)); + ppEnd = pGroup->sorted_servers + pGroup->count; + for (ppServer=pGroup->sorted_servers; ppServerid) != 0) + { + continue; + } + + pStatBuff = &(pDest->stat_buff); + pStorageStat = &((*ppServer)->stat); + pDest->status = (*ppServer)->status; + strcpy(pDest->id, (*ppServer)->id); + strcpy(pDest->ip_addr, (*ppServer)->ip_addr); + if ((*ppServer)->psync_src_server != NULL) + { + strcpy(pDest->src_id, \ + (*ppServer)->psync_src_server->id); + } + + strcpy(pDest->domain_name, (*ppServer)->domain_name); + strcpy(pDest->version, (*ppServer)->version); + long2buff((*ppServer)->join_time, pDest->sz_join_time); + long2buff((*ppServer)->up_time, pDest->sz_up_time); + long2buff((*ppServer)->total_mb, pDest->sz_total_mb); + long2buff((*ppServer)->free_mb, pDest->sz_free_mb); + long2buff((*ppServer)->upload_priority, \ + pDest->sz_upload_priority); + long2buff((*ppServer)->storage_port, \ + pDest->sz_storage_port); + long2buff((*ppServer)->storage_http_port, \ + pDest->sz_storage_http_port); + long2buff((*ppServer)->store_path_count, \ + pDest->sz_store_path_count); + long2buff((*ppServer)->subdir_count_per_path, \ + pDest->sz_subdir_count_per_path); + long2buff((*ppServer)->current_write_path, \ + pDest->sz_current_write_path); + + long2buff(pStorageStat->total_upload_count, \ + pStatBuff->sz_total_upload_count); + long2buff(pStorageStat->success_upload_count, \ + pStatBuff->sz_success_upload_count); + long2buff(pStorageStat->total_append_count, \ + pStatBuff->sz_total_append_count); + long2buff(pStorageStat->success_append_count, \ + pStatBuff->sz_success_append_count); + long2buff(pStorageStat->total_modify_count, \ + pStatBuff->sz_total_modify_count); + long2buff(pStorageStat->success_modify_count, \ + pStatBuff->sz_success_modify_count); + long2buff(pStorageStat->total_truncate_count, \ + pStatBuff->sz_total_truncate_count); + long2buff(pStorageStat->success_truncate_count, \ + pStatBuff->sz_success_truncate_count); + long2buff(pStorageStat->total_set_meta_count, \ + pStatBuff->sz_total_set_meta_count); + long2buff(pStorageStat->success_set_meta_count, \ + pStatBuff->sz_success_set_meta_count); + long2buff(pStorageStat->total_delete_count, \ + pStatBuff->sz_total_delete_count); + long2buff(pStorageStat->success_delete_count, \ + pStatBuff->sz_success_delete_count); + long2buff(pStorageStat->total_download_count, \ + pStatBuff->sz_total_download_count); + long2buff(pStorageStat->success_download_count, \ + pStatBuff->sz_success_download_count); + long2buff(pStorageStat->total_get_meta_count, \ + pStatBuff->sz_total_get_meta_count); + long2buff(pStorageStat->success_get_meta_count, \ + pStatBuff->sz_success_get_meta_count); + long2buff(pStorageStat->last_source_update, \ + pStatBuff->sz_last_source_update); + long2buff(pStorageStat->last_sync_update, \ + pStatBuff->sz_last_sync_update); + long2buff(pStorageStat->last_synced_timestamp, \ + pStatBuff->sz_last_synced_timestamp); + long2buff(pStorageStat->total_create_link_count, \ + pStatBuff->sz_total_create_link_count); + long2buff(pStorageStat->success_create_link_count, \ + pStatBuff->sz_success_create_link_count); + long2buff(pStorageStat->total_delete_link_count, \ + pStatBuff->sz_total_delete_link_count); + long2buff(pStorageStat->success_delete_link_count, \ + pStatBuff->sz_success_delete_link_count); + long2buff(pStorageStat->total_upload_bytes, \ + pStatBuff->sz_total_upload_bytes); + long2buff(pStorageStat->success_upload_bytes, \ + pStatBuff->sz_success_upload_bytes); + long2buff(pStorageStat->total_append_bytes, \ + pStatBuff->sz_total_append_bytes); + long2buff(pStorageStat->success_append_bytes, \ + pStatBuff->sz_success_append_bytes); + long2buff(pStorageStat->total_modify_bytes, \ + pStatBuff->sz_total_modify_bytes); + long2buff(pStorageStat->success_modify_bytes, \ + pStatBuff->sz_success_modify_bytes); + long2buff(pStorageStat->total_download_bytes, \ + pStatBuff->sz_total_download_bytes); + long2buff(pStorageStat->success_download_bytes, \ + pStatBuff->sz_success_download_bytes); + long2buff(pStorageStat->total_sync_in_bytes, \ + pStatBuff->sz_total_sync_in_bytes); + long2buff(pStorageStat->success_sync_in_bytes, \ + pStatBuff->sz_success_sync_in_bytes); + long2buff(pStorageStat->total_sync_out_bytes, \ + pStatBuff->sz_total_sync_out_bytes); + long2buff(pStorageStat->success_sync_out_bytes, \ + pStatBuff->sz_success_sync_out_bytes); + long2buff(pStorageStat->total_file_open_count, \ + pStatBuff->sz_total_file_open_count); + long2buff(pStorageStat->success_file_open_count, \ + pStatBuff->sz_success_file_open_count); + long2buff(pStorageStat->total_file_read_count, \ + pStatBuff->sz_total_file_read_count); + long2buff(pStorageStat->success_file_read_count, \ + pStatBuff->sz_success_file_read_count); + long2buff(pStorageStat->total_file_write_count, \ + pStatBuff->sz_total_file_write_count); + long2buff(pStorageStat->success_file_write_count, \ + pStatBuff->sz_success_file_write_count); + long2buff(pStorageStat->last_heart_beat_time, \ + pStatBuff->sz_last_heart_beat_time); + pDest->if_trunk_server = (pGroup->pTrunkServer == *ppServer); + + pDest++; + } + + if (pStorageId != NULL && pDest - pStart == 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pTask->length = sizeof(TrackerHeader) + (pDest - pStart) * \ + sizeof(TrackerStorageStat); + return 0; +} + +/** +pkg format: +Header +FDFS_GROUP_NAME_MAX_LEN bytes: group_name +remain bytes: filename +**/ +static int tracker_deal_service_query_fetch_update( \ + struct fast_task_info *pTask, const byte cmd) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char *filename; + char *p; + FDFSGroupInfo *pGroup; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + FDFSStorageDetail *ppStoreServers[FDFS_MAX_SERVERS_EACH_GROUP]; + int filename_len; + int server_count; + int result; + int nPkgLen; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen < FDFS_GROUP_NAME_MAX_LEN+22) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length > %d", __LINE__, cmd, \ + pTask->client_ip, nPkgLen, \ + FDFS_GROUP_NAME_MAX_LEN+22); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + if (nPkgLen >= FDFS_GROUP_NAME_MAX_LEN + 128) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is too long, exceeds %d", \ + __LINE__, cmd, pTask->client_ip, nPkgLen, \ + FDFS_GROUP_NAME_MAX_LEN + 128); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->data[pTask->length] = '\0'; + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0'; + filename = pTask->data + sizeof(TrackerHeader)+FDFS_GROUP_NAME_MAX_LEN; + filename_len = pTask->length - sizeof(TrackerHeader) - \ + FDFS_GROUP_NAME_MAX_LEN; + + result = tracker_mem_get_storage_by_filename(cmd, \ + FDFS_DOWNLOAD_TYPE_CALL \ + group_name, filename, filename_len, &pGroup, \ + ppStoreServers, &server_count); + + if (result != 0) + { + pTask->length = sizeof(TrackerHeader); + return result; + } + + + pTask->length = sizeof(TrackerHeader) + \ + TRACKER_QUERY_STORAGE_FETCH_BODY_LEN + \ + (server_count - 1) * (IP_ADDRESS_SIZE - 1); + + p = pTask->data + sizeof(TrackerHeader); + memcpy(p, pGroup->group_name, FDFS_GROUP_NAME_MAX_LEN); + p += FDFS_GROUP_NAME_MAX_LEN; + memcpy(p, ppStoreServers[0]->ip_addr, IP_ADDRESS_SIZE-1); + p += IP_ADDRESS_SIZE - 1; + long2buff(pGroup->storage_port, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + + if (server_count > 1) + { + ppServerEnd = ppStoreServers + server_count; + for (ppServer=ppStoreServers+1; ppServerip_addr, \ + IP_ADDRESS_SIZE - 1); + p += IP_ADDRESS_SIZE - 1; + } + } + + return 0; +} + +#define tracker_check_reserved_space(pGroup) \ + fdfs_check_reserved_space(pGroup, &g_storage_reserved_space) + +#define tracker_check_reserved_space_trunk(pGroup) \ + fdfs_check_reserved_space_trunk(pGroup, &g_storage_reserved_space) + +#define tracker_check_reserved_space_path(total_mb, free_mb, avg_mb) \ + fdfs_check_reserved_space_path(total_mb, free_mb, avg_mb, \ + &g_storage_reserved_space) + +static int tracker_deal_service_query_storage( \ + struct fast_task_info *pTask, char cmd) +{ + int expect_pkg_len; + FDFSGroupInfo *pStoreGroup; + FDFSGroupInfo **ppFoundGroup; + FDFSGroupInfo **ppGroup; + FDFSStorageDetail *pStorageServer; + char *group_name; + char *p; + bool bHaveActiveServer; + int write_path_index; + int avg_reserved_mb; + + if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE + || cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL) + { + expect_pkg_len = FDFS_GROUP_NAME_MAX_LEN; + } + else + { + expect_pkg_len = 0; + } + + if (pTask->length - sizeof(TrackerHeader) != expect_pkg_len) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT"is not correct, " \ + "expect length: %d", __LINE__, \ + cmd, pTask->client_ip, \ + pTask->length - (int)sizeof(TrackerHeader), \ + expect_pkg_len); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + if (g_groups.count == 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE + || cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL) + { + group_name = pTask->data + sizeof(TrackerHeader); + group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0'; + + pStoreGroup = tracker_mem_get_group(group_name); + if (pStoreGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (pStoreGroup->active_count == 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (!tracker_check_reserved_space(pStoreGroup)) + { + if (!(g_if_use_trunk_file && \ + tracker_check_reserved_space_trunk(pStoreGroup))) + { + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + } + } + else if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN + ||g_groups.store_lookup==FDFS_STORE_LOOKUP_LOAD_BALANCE) + { + int write_group_index; + + bHaveActiveServer = false; + write_group_index = g_groups.current_write_group; + if (write_group_index >= g_groups.count) + { + write_group_index = 0; + } + + pStoreGroup = NULL; + ppFoundGroup = g_groups.sorted_groups + write_group_index; + if ((*ppFoundGroup)->active_count > 0) + { + bHaveActiveServer = true; + if (tracker_check_reserved_space(*ppFoundGroup)) + { + pStoreGroup = *ppFoundGroup; + } + else if (g_if_use_trunk_file && \ + g_groups.store_lookup == \ + FDFS_STORE_LOOKUP_LOAD_BALANCE && \ + tracker_check_reserved_space_trunk( \ + *ppFoundGroup)) + { + pStoreGroup = *ppFoundGroup; + } + } + + if (pStoreGroup == NULL) + { + FDFSGroupInfo **ppGroupEnd; + ppGroupEnd = g_groups.sorted_groups + \ + g_groups.count; + for (ppGroup=ppFoundGroup+1; \ + ppGroupactive_count == 0) + { + continue; + } + + bHaveActiveServer = true; + if (tracker_check_reserved_space(*ppGroup)) + { + pStoreGroup = *ppGroup; + g_groups.current_write_group = \ + ppGroup-g_groups.sorted_groups; + break; + } + } + + if (pStoreGroup == NULL) + { + for (ppGroup=g_groups.sorted_groups; \ + ppGroupactive_count == 0) + { + continue; + } + + bHaveActiveServer = true; + if (tracker_check_reserved_space(*ppGroup)) + { + pStoreGroup = *ppGroup; + g_groups.current_write_group = \ + ppGroup-g_groups.sorted_groups; + break; + } + } + } + + if (pStoreGroup == NULL) + { + if (!bHaveActiveServer) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (!g_if_use_trunk_file) + { + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + + for (ppGroup=g_groups.sorted_groups; \ + ppGroupactive_count == 0) + { + continue; + } + if (tracker_check_reserved_space_trunk(*ppGroup)) + { + pStoreGroup = *ppGroup; + g_groups.current_write_group = \ + ppGroup-g_groups.sorted_groups; + break; + } + } + + if (pStoreGroup == NULL) + { + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + } + } + + if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN) + { + g_groups.current_write_group++; + if (g_groups.current_write_group >= g_groups.count) + { + g_groups.current_write_group = 0; + } + } + } + else if (g_groups.store_lookup == FDFS_STORE_LOOKUP_SPEC_GROUP) + { + if (g_groups.pStoreGroup == NULL || \ + g_groups.pStoreGroup->active_count == 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (!tracker_check_reserved_space(g_groups.pStoreGroup)) + { + if (!(g_if_use_trunk_file && \ + tracker_check_reserved_space_trunk( \ + g_groups.pStoreGroup))) + { + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + } + + pStoreGroup = g_groups.pStoreGroup; + } + else + { + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + if (pStoreGroup->store_path_count <= 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE + || cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE) + { + pStorageServer = tracker_get_writable_storage(pStoreGroup); + if (pStorageServer == NULL) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + } + else //query store server list, use the first to check + { + pStorageServer = *(pStoreGroup->active_servers); + } + + write_path_index = pStorageServer->current_write_path; + if (write_path_index >= pStoreGroup->store_path_count) + { + write_path_index = 0; + } + + avg_reserved_mb = g_storage_reserved_space.rs.mb / \ + pStoreGroup->store_path_count; + if (!tracker_check_reserved_space_path(pStorageServer-> \ + path_total_mbs[write_path_index], pStorageServer-> \ + path_free_mbs[write_path_index], avg_reserved_mb)) + { + int i; + for (i=0; istore_path_count; i++) + { + if (tracker_check_reserved_space_path( \ + pStorageServer->path_total_mbs[i], \ + pStorageServer->path_free_mbs[i], \ + avg_reserved_mb)) + { + pStorageServer->current_write_path = i; + write_path_index = i; + break; + } + } + + if (i == pStoreGroup->store_path_count) + { + if (!g_if_use_trunk_file) + { + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + + for (i=write_path_index; i \ + store_path_count; i++) + { + if (tracker_check_reserved_space_path( \ + pStorageServer->path_total_mbs[i], \ + pStorageServer->path_free_mbs[i] + \ + pStoreGroup->trunk_free_mb, \ + avg_reserved_mb)) + { + pStorageServer->current_write_path = i; + write_path_index = i; + break; + } + } + if ( i == pStoreGroup->store_path_count) + { + for (i=0; ipath_total_mbs[i], \ + pStorageServer->path_free_mbs[i] + \ + pStoreGroup->trunk_free_mb, \ + avg_reserved_mb)) + { + pStorageServer->current_write_path = i; + write_path_index = i; + break; + } + } + + if (i == write_path_index) + { + pTask->length = sizeof(TrackerHeader); + return ENOSPC; + } + } + } + } + + if (g_groups.store_path == FDFS_STORE_PATH_ROUND_ROBIN) + { + pStorageServer->current_write_path++; + if (pStorageServer->current_write_path >= \ + pStoreGroup->store_path_count) + { + pStorageServer->current_write_path = 0; + } + } + + /* + //printf("pStoreGroup->current_write_server: %d, " \ + "pStoreGroup->active_count=%d\n", \ + pStoreGroup->current_write_server, \ + pStoreGroup->active_count); + */ + + p = pTask->data + sizeof(TrackerHeader); + memcpy(p, pStoreGroup->group_name, FDFS_GROUP_NAME_MAX_LEN); + p += FDFS_GROUP_NAME_MAX_LEN; + + if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL + || cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL) + { + int active_count; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppEnd; + + active_count = pStoreGroup->active_count; + if (active_count == 0) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + ppEnd = pStoreGroup->active_servers + active_count; + for (ppServer=pStoreGroup->active_servers; ppServerip_addr, IP_ADDRESS_SIZE - 1); + p += IP_ADDRESS_SIZE - 1; + + long2buff(pStoreGroup->storage_port, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + } + } + else + { + memcpy(p, pStorageServer->ip_addr, IP_ADDRESS_SIZE - 1); + p += IP_ADDRESS_SIZE - 1; + + long2buff(pStoreGroup->storage_port, p); + p += FDFS_PROTO_PKG_LEN_SIZE; + } + + *p++ = (char)write_path_index; + + pTask->length = p - pTask->data; + + return 0; +} + +static int tracker_deal_server_list_one_group(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSGroupInfo *pGroup; + TrackerGroupStat *pDest; + + if (pTask->length - sizeof(TrackerHeader) != FDFS_GROUP_NAME_MAX_LEN) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), FDFS_GROUP_NAME_MAX_LEN); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, group name: %s not exist", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pDest = (TrackerGroupStat *)(pTask->data + sizeof(TrackerHeader)); + memcpy(pDest->group_name, pGroup->group_name, \ + FDFS_GROUP_NAME_MAX_LEN + 1); + long2buff(pGroup->total_mb, pDest->sz_total_mb); + long2buff(pGroup->free_mb, pDest->sz_free_mb); + long2buff(pGroup->trunk_free_mb, pDest->sz_trunk_free_mb); + long2buff(pGroup->count, pDest->sz_count); + long2buff(pGroup->storage_port, pDest->sz_storage_port); + long2buff(pGroup->storage_http_port, pDest->sz_storage_http_port); + long2buff(pGroup->active_count, pDest->sz_active_count); + long2buff(pGroup->current_write_server, + pDest->sz_current_write_server); + long2buff(pGroup->store_path_count, pDest->sz_store_path_count); + long2buff(pGroup->subdir_count_per_path, \ + pDest->sz_subdir_count_per_path); + long2buff(pGroup->current_trunk_file_id, \ + pDest->sz_current_trunk_file_id); + pTask->length = sizeof(TrackerHeader) + sizeof(TrackerGroupStat); + + return 0; +} + +static int tracker_deal_server_list_all_groups(struct fast_task_info *pTask) +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppEnd; + TrackerGroupStat *groupStats; + TrackerGroupStat *pDest; + + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: 0", __LINE__, \ + TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + groupStats = (TrackerGroupStat *)(pTask->data + sizeof(TrackerHeader)); + pDest = groupStats; + ppEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; ppGroupgroup_name, (*ppGroup)->group_name, \ + FDFS_GROUP_NAME_MAX_LEN + 1); + long2buff((*ppGroup)->total_mb, pDest->sz_total_mb); + long2buff((*ppGroup)->free_mb, pDest->sz_free_mb); + long2buff((*ppGroup)->trunk_free_mb, pDest->sz_trunk_free_mb); + long2buff((*ppGroup)->count, pDest->sz_count); + long2buff((*ppGroup)->storage_port, \ + pDest->sz_storage_port); + long2buff((*ppGroup)->storage_http_port, \ + pDest->sz_storage_http_port); + long2buff((*ppGroup)->active_count, \ + pDest->sz_active_count); + long2buff((*ppGroup)->current_write_server, \ + pDest->sz_current_write_server); + long2buff((*ppGroup)->store_path_count, \ + pDest->sz_store_path_count); + long2buff((*ppGroup)->subdir_count_per_path, \ + pDest->sz_subdir_count_per_path); + long2buff((*ppGroup)->current_trunk_file_id, \ + pDest->sz_current_trunk_file_id); + pDest++; + } + + pTask->length = sizeof(TrackerHeader) + (pDest - groupStats) * \ + sizeof(TrackerGroupStat); + + return 0; +} + +static int tracker_deal_storage_sync_src_req(struct fast_task_info *pTask) +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + FDFSGroupInfo *pGroup; + char *dest_storage_id; + FDFSStorageDetail *pDestStorage; + + if (pTask->length - sizeof(TrackerHeader) != \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_SYNC_SRC_REQ, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN + FDFS_STORAGE_ID_MAX_SIZE); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + memcpy(group_name, pTask->data + sizeof(TrackerHeader), \ + FDFS_GROUP_NAME_MAX_LEN); + *(group_name + FDFS_GROUP_NAME_MAX_LEN) = '\0'; + pGroup = tracker_mem_get_group(group_name); + if (pGroup == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, invalid group_name: %s", \ + __LINE__, pTask->client_ip, group_name); + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + dest_storage_id = pTask->data + sizeof(TrackerHeader) + \ + FDFS_GROUP_NAME_MAX_LEN; + dest_storage_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0'; + pDestStorage = tracker_mem_get_storage(pGroup, dest_storage_id); + if (pDestStorage == NULL) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + if (pDestStorage->status == FDFS_STORAGE_STATUS_INIT || \ + pDestStorage->status == FDFS_STORAGE_STATUS_DELETED || \ + pDestStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pTask->length = sizeof(TrackerHeader); + if (pDestStorage->psync_src_server != NULL) + { + if (pDestStorage->psync_src_server->status == \ + FDFS_STORAGE_STATUS_OFFLINE \ + || pDestStorage->psync_src_server->status == \ + FDFS_STORAGE_STATUS_ONLINE \ + || pDestStorage->psync_src_server->status == \ + FDFS_STORAGE_STATUS_ACTIVE \ + || pDestStorage->psync_src_server->status == \ + FDFS_STORAGE_STATUS_RECOVERY) + { + TrackerStorageSyncReqBody *pBody; + pBody = (TrackerStorageSyncReqBody *)(pTask->data + \ + sizeof(TrackerHeader)); + strcpy(pBody->src_id, \ + pDestStorage->psync_src_server->id); + long2buff(pDestStorage->sync_until_timestamp, \ + pBody->until_timestamp); + pTask->length += sizeof(TrackerStorageSyncReqBody); + } + else + { + pDestStorage->psync_src_server = NULL; + tracker_save_storages(); + } + } + + return 0; +} + +static int tracker_deal_storage_sync_dest_req(struct fast_task_info *pTask) +{ + TrackerStorageSyncReqBody *pBody; + FDFSStorageDetail *pSrcStorage; + FDFSStorageDetail **ppServer; + FDFSStorageDetail **ppServerEnd; + int sync_until_timestamp; + int source_count; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + pSrcStorage = NULL; + sync_until_timestamp = (int)g_current_time; + + do + { + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: 0", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + if (pClientInfo->pGroup->count <= 1) + { + break; + } + + source_count = 0; + ppServerEnd = pClientInfo->pGroup->all_servers + \ + pClientInfo->pGroup->count; + for (ppServer=pClientInfo->pGroup->all_servers; \ + ppServerid, \ + pClientInfo->pStorage->id) == 0) + { + continue; + } + + if ((*ppServer)->status ==FDFS_STORAGE_STATUS_OFFLINE + || (*ppServer)->status == FDFS_STORAGE_STATUS_ONLINE + || (*ppServer)->status == FDFS_STORAGE_STATUS_ACTIVE) + { + source_count++; + } + } + if (source_count == 0) + { + break; + } + + pSrcStorage = tracker_get_group_sync_src_server( \ + pClientInfo->pGroup, pClientInfo->pStorage); + if (pSrcStorage == NULL) + { + pTask->length = sizeof(TrackerHeader); + return ENOENT; + } + + pBody=(TrackerStorageSyncReqBody *)(pTask->data+sizeof(TrackerHeader)); + strcpy(pBody->src_id, pSrcStorage->id); + long2buff(sync_until_timestamp, pBody->until_timestamp); + + } while (0); + + if (pSrcStorage == NULL) + { + pClientInfo->pStorage->status = \ + FDFS_STORAGE_STATUS_ONLINE; + pClientInfo->pGroup->chg_count++; + tracker_save_storages(); + + pTask->length = sizeof(TrackerHeader); + return 0; + } + + pClientInfo->pStorage->psync_src_server = pSrcStorage; + pClientInfo->pStorage->sync_until_timestamp = sync_until_timestamp; + pClientInfo->pStorage->status = FDFS_STORAGE_STATUS_WAIT_SYNC; + pClientInfo->pGroup->chg_count++; + + tracker_save_storages(); + + pTask->length = sizeof(TrackerHeader)+sizeof(TrackerStorageSyncReqBody); + return 0; +} + +static int tracker_deal_storage_sync_dest_query(struct fast_task_info *pTask) +{ + FDFSStorageDetail *pSrcStorage; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + if (pTask->length - sizeof(TrackerHeader) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: 0", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_QUERY, \ + pTask->client_ip, pTask->length - \ + (int)sizeof(TrackerHeader)); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + pTask->length = sizeof(TrackerHeader); + pSrcStorage = pClientInfo->pStorage->psync_src_server; + if (pSrcStorage != NULL) + { + TrackerStorageSyncReqBody *pBody; + pBody = (TrackerStorageSyncReqBody *)(pTask->data + \ + sizeof(TrackerHeader)); + strcpy(pBody->src_id, pSrcStorage->id); + + long2buff(pClientInfo->pStorage->sync_until_timestamp, \ + pBody->until_timestamp); + pTask->length += sizeof(TrackerStorageSyncReqBody); + } + + + return 0; +} + +static void tracker_find_max_free_space_group() +{ + FDFSGroupInfo **ppGroup; + FDFSGroupInfo **ppGroupEnd; + FDFSGroupInfo **ppMaxGroup; + int result; + + if (g_groups.store_lookup != FDFS_STORE_LOOKUP_LOAD_BALANCE) + { + return; + } + + if ((result=pthread_mutex_lock(&lb_thread_lock)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "call pthread_mutex_lock fail, " \ + "errno: %d, error info: %s", \ + __LINE__, result, STRERROR(result)); + } + + ppMaxGroup = NULL; + ppGroupEnd = g_groups.sorted_groups + g_groups.count; + for (ppGroup=g_groups.sorted_groups; \ + ppGroupactive_count > 0) + { + if (ppMaxGroup == NULL) + { + ppMaxGroup = ppGroup; + } + else if ((*ppGroup)->free_mb > (*ppMaxGroup)->free_mb) + { + ppMaxGroup = ppGroup; + } + } + } + + if (ppMaxGroup == NULL) + { + pthread_mutex_unlock(&lb_thread_lock); + return; + } + + if (tracker_check_reserved_space(*ppMaxGroup) \ + || !g_if_use_trunk_file) + { + g_groups.current_write_group = \ + ppMaxGroup - g_groups.sorted_groups; + pthread_mutex_unlock(&lb_thread_lock); + return; + } + + ppMaxGroup = NULL; + for (ppGroup=g_groups.sorted_groups; \ + ppGroupactive_count > 0) + { + if (ppMaxGroup == NULL) + { + ppMaxGroup = ppGroup; + } + else if ((*ppGroup)->trunk_free_mb > \ + (*ppMaxGroup)->trunk_free_mb) + { + ppMaxGroup = ppGroup; + } + } + } + + if (ppMaxGroup == NULL) + { + pthread_mutex_unlock(&lb_thread_lock); + return; + } + + g_groups.current_write_group = \ + ppMaxGroup - g_groups.sorted_groups; + pthread_mutex_unlock(&lb_thread_lock); +} + +static void tracker_find_min_free_space(FDFSGroupInfo *pGroup) +{ + FDFSStorageDetail **ppServerEnd; + FDFSStorageDetail **ppServer; + + if (pGroup->active_count == 0) + { + return; + } + + pGroup->total_mb = (*(pGroup->active_servers))->total_mb; + pGroup->free_mb = (*(pGroup->active_servers))->free_mb; + ppServerEnd = pGroup->active_servers + pGroup->active_count; + for (ppServer=pGroup->active_servers+1; \ + ppServerfree_mb < pGroup->free_mb) + { + pGroup->total_mb = (*ppServer)->total_mb; + pGroup->free_mb = (*ppServer)->free_mb; + } + } +} + +static int tracker_deal_storage_sync_report(struct fast_task_info *pTask) +{ + char *p; + char *pEnd; + char *src_id; + int status; + int sync_timestamp; + int src_index; + int dest_index; + int nPkgLen; + FDFSStorageDetail *pSrcStorage; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen <= 0 || nPkgLen % (FDFS_STORAGE_ID_MAX_SIZE + 4) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT, \ + pTask->client_ip, nPkgLen); + + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + do + { + dest_index = tracker_mem_get_storage_index(pClientInfo->pGroup, + pClientInfo->pStorage); + if (dest_index < 0 || dest_index >= pClientInfo->pGroup->count) + { + status = 0; + break; + } + + if (g_groups.store_server == FDFS_STORE_SERVER_ROUND_ROBIN) + { + int min_synced_timestamp; + + min_synced_timestamp = 0; + pEnd = pTask->data + pTask->length; + for (p=pTask->data + sizeof(TrackerHeader); ppGroup, src_id); + if (pSrcStorage == NULL) + { + continue; + } + if (pSrcStorage->status != FDFS_STORAGE_STATUS_ACTIVE) + { + continue; + } + + src_index = tracker_mem_get_storage_index( \ + pClientInfo->pGroup, pSrcStorage); + if (src_index == dest_index || src_index < 0 || \ + src_index >= pClientInfo->pGroup->count) + { + continue; + } + + pClientInfo->pGroup->last_sync_timestamps \ + [src_index][dest_index] = sync_timestamp; + + if (min_synced_timestamp == 0) + { + min_synced_timestamp = sync_timestamp; + } + else if (sync_timestamp < min_synced_timestamp) + { + min_synced_timestamp = sync_timestamp; + } + } + + if (min_synced_timestamp > 0) + { + pClientInfo->pStorage->stat.last_synced_timestamp = \ + min_synced_timestamp; + } + } + else + { + int max_synced_timestamp; + + max_synced_timestamp = pClientInfo->pStorage->stat.\ + last_synced_timestamp; + pEnd = pTask->data + pTask->length; + for (p=pTask->data + sizeof(TrackerHeader); ppGroup, src_id); + if (pSrcStorage == NULL) + { + continue; + } + if (pSrcStorage->status != FDFS_STORAGE_STATUS_ACTIVE) + { + continue; + } + + src_index = tracker_mem_get_storage_index( \ + pClientInfo->pGroup, pSrcStorage); + if (src_index == dest_index || src_index < 0 || \ + src_index >= pClientInfo->pGroup->count) + { + continue; + } + + pClientInfo->pGroup->last_sync_timestamps \ + [src_index][dest_index] = sync_timestamp; + + if (sync_timestamp > max_synced_timestamp) + { + max_synced_timestamp = sync_timestamp; + } + } + + pClientInfo->pStorage->stat.last_synced_timestamp = \ + max_synced_timestamp; + } + + if (++g_storage_sync_time_chg_count % \ + TRACKER_SYNC_TO_FILE_FREQ == 0) + { + status = tracker_save_sync_timestamps(); + } + else + { + status = 0; + } + } while (0); + + return tracker_check_and_sync(pTask, status); +} + +static int tracker_deal_storage_df_report(struct fast_task_info *pTask) +{ + int nPkgLen; + int i; + TrackerStatReportReqBody *pStatBuff; + int64_t *path_total_mbs; + int64_t *path_free_mbs; + int64_t old_free_mb; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + if (pClientInfo->pGroup == NULL || pClientInfo->pStorage == NULL) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, not join in!", \ + __LINE__, TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE, \ + pTask->client_ip); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen != sizeof(TrackerStatReportReqBody) * \ + pClientInfo->pGroup->store_path_count) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE, \ + pTask->client_ip, nPkgLen, \ + (int)sizeof(TrackerStatReportReqBody) * \ + pClientInfo->pGroup->store_path_count); + pTask->length = sizeof(TrackerHeader); + return EINVAL; + } + + old_free_mb = pClientInfo->pStorage->free_mb; + path_total_mbs = pClientInfo->pStorage->path_total_mbs; + path_free_mbs = pClientInfo->pStorage->path_free_mbs; + pClientInfo->pStorage->total_mb = 0; + pClientInfo->pStorage->free_mb = 0; + + pStatBuff = (TrackerStatReportReqBody *)(pTask->data + sizeof(TrackerHeader)); + for (i=0; ipGroup->store_path_count; i++) + { + path_total_mbs[i] = buff2long(pStatBuff->sz_total_mb); + path_free_mbs[i] = buff2long(pStatBuff->sz_free_mb); + + pClientInfo->pStorage->total_mb += path_total_mbs[i]; + pClientInfo->pStorage->free_mb += path_free_mbs[i]; + + if (g_groups.store_path == FDFS_STORE_PATH_LOAD_BALANCE + && path_free_mbs[i] > path_free_mbs[ \ + pClientInfo->pStorage->current_write_path]) + { + pClientInfo->pStorage->current_write_path = i; + } + + pStatBuff++; + } + + if ((pClientInfo->pGroup->free_mb == 0) || + (pClientInfo->pStorage->free_mb < pClientInfo->pGroup->free_mb)) + { + pClientInfo->pGroup->total_mb = pClientInfo->pStorage->total_mb; + pClientInfo->pGroup->free_mb = pClientInfo->pStorage->free_mb; + } + else if (pClientInfo->pStorage->free_mb > old_free_mb) + { + tracker_find_min_free_space(pClientInfo->pGroup); + } + + tracker_find_max_free_space_group(); + + /* + //logInfo("storage: %s:%d, total_mb=%dMB, free_mb=%dMB\n", \ + pClientInfo->pStorage->ip_addr, \ + pClientInfo->pGroup->storage_port, \ + pClientInfo->pStorage->total_mb, \ + pClientInfo->pStorage->free_mb); + */ + + tracker_mem_active_store_server(pClientInfo->pGroup, \ + pClientInfo->pStorage); + + return tracker_check_and_sync(pTask, 0); +} + +static int tracker_deal_storage_beat(struct fast_task_info *pTask) +{ + int nPkgLen; + int status; + FDFSStorageStatBuff *pStatBuff; + FDFSStorageStat *pStat; + TrackerClientInfo *pClientInfo; + + pClientInfo = (TrackerClientInfo *)pTask->arg; + + do + { + nPkgLen = pTask->length - sizeof(TrackerHeader); + if (nPkgLen == 0) + { + status = 0; + break; + } + + if (nPkgLen != sizeof(FDFSStorageStatBuff)) + { + logError("file: "__FILE__", line: %d, " \ + "cmd=%d, client ip: %s, package size " \ + PKG_LEN_PRINTF_FORMAT" is not correct, " \ + "expect length: 0 or %d", __LINE__, \ + TRACKER_PROTO_CMD_STORAGE_BEAT, \ + pTask->client_ip, nPkgLen, + (int)sizeof(FDFSStorageStatBuff)); + status = EINVAL; + break; + } + + pStatBuff = (FDFSStorageStatBuff *)(pTask->data + \ + sizeof(TrackerHeader)); + pStat = &(pClientInfo->pStorage->stat); + + pStat->total_upload_count = \ + buff2long(pStatBuff->sz_total_upload_count); + pStat->success_upload_count = \ + buff2long(pStatBuff->sz_success_upload_count); + pStat->total_append_count = \ + buff2long(pStatBuff->sz_total_append_count); + pStat->success_append_count = \ + buff2long(pStatBuff->sz_success_append_count); + pStat->total_modify_count = \ + buff2long(pStatBuff->sz_total_modify_count); + pStat->success_modify_count = \ + buff2long(pStatBuff->sz_success_modify_count); + pStat->total_truncate_count = \ + buff2long(pStatBuff->sz_total_truncate_count); + pStat->success_truncate_count = \ + buff2long(pStatBuff->sz_success_truncate_count); + pStat->total_download_count = \ + buff2long(pStatBuff->sz_total_download_count); + pStat->success_download_count = \ + buff2long(pStatBuff->sz_success_download_count); + pStat->total_set_meta_count = \ + buff2long(pStatBuff->sz_total_set_meta_count); + pStat->success_set_meta_count = \ + buff2long(pStatBuff->sz_success_set_meta_count); + pStat->total_delete_count = \ + buff2long(pStatBuff->sz_total_delete_count); + pStat->success_delete_count = \ + buff2long(pStatBuff->sz_success_delete_count); + pStat->total_get_meta_count = \ + buff2long(pStatBuff->sz_total_get_meta_count); + pStat->success_get_meta_count = \ + buff2long(pStatBuff->sz_success_get_meta_count); + pStat->last_source_update = \ + buff2long(pStatBuff->sz_last_source_update); + pStat->last_sync_update = \ + buff2long(pStatBuff->sz_last_sync_update); + pStat->total_create_link_count = \ + buff2long(pStatBuff->sz_total_create_link_count); + pStat->success_create_link_count = \ + buff2long(pStatBuff->sz_success_create_link_count); + pStat->total_delete_link_count = \ + buff2long(pStatBuff->sz_total_delete_link_count); + pStat->success_delete_link_count = \ + buff2long(pStatBuff->sz_success_delete_link_count); + pStat->total_upload_bytes = \ + buff2long(pStatBuff->sz_total_upload_bytes); + pStat->success_upload_bytes = \ + buff2long(pStatBuff->sz_success_upload_bytes); + pStat->total_append_bytes = \ + buff2long(pStatBuff->sz_total_append_bytes); + pStat->success_append_bytes = \ + buff2long(pStatBuff->sz_success_append_bytes); + pStat->total_modify_bytes = \ + buff2long(pStatBuff->sz_total_modify_bytes); + pStat->success_modify_bytes = \ + buff2long(pStatBuff->sz_success_modify_bytes); + pStat->total_download_bytes = \ + buff2long(pStatBuff->sz_total_download_bytes); + pStat->success_download_bytes = \ + buff2long(pStatBuff->sz_success_download_bytes); + pStat->total_sync_in_bytes = \ + buff2long(pStatBuff->sz_total_sync_in_bytes); + pStat->success_sync_in_bytes = \ + buff2long(pStatBuff->sz_success_sync_in_bytes); + pStat->total_sync_out_bytes = \ + buff2long(pStatBuff->sz_total_sync_out_bytes); + pStat->success_sync_out_bytes = \ + buff2long(pStatBuff->sz_success_sync_out_bytes); + pStat->total_file_open_count = \ + buff2long(pStatBuff->sz_total_file_open_count); + pStat->success_file_open_count = \ + buff2long(pStatBuff->sz_success_file_open_count); + pStat->total_file_read_count = \ + buff2long(pStatBuff->sz_total_file_read_count); + pStat->success_file_read_count = \ + buff2long(pStatBuff->sz_success_file_read_count); + pStat->total_file_write_count = \ + buff2long(pStatBuff->sz_total_file_write_count); + pStat->success_file_write_count = \ + buff2long(pStatBuff->sz_success_file_write_count); + + if (++g_storage_stat_chg_count % TRACKER_SYNC_TO_FILE_FREQ == 0) + { + status = tracker_save_storages(); + } + else + { + status = 0; + } + + //printf("g_storage_stat_chg_count=%d\n", g_storage_stat_chg_count); + + } while (0); + + if (status == 0) + { + tracker_mem_active_store_server(pClientInfo->pGroup, \ + pClientInfo->pStorage); + pClientInfo->pStorage->stat.last_heart_beat_time = g_current_time; + + } + + //printf("deal heart beat, status=%d\n", status); + return tracker_check_and_sync(pTask, status); +} + +#define TRACKER_CHECK_LOGINED(pTask) \ + if (((TrackerClientInfo *)pTask->arg)->pGroup == NULL || \ + ((TrackerClientInfo *)pTask->arg)->pStorage == NULL) \ + { \ + pTask->length = sizeof(TrackerHeader); \ + result = EACCES; \ + break; \ + } \ + + +int tracker_deal_task(struct fast_task_info *pTask) +{ + TrackerHeader *pHeader; + int result; + + pHeader = (TrackerHeader *)pTask->data; + switch(pHeader->cmd) + { + case TRACKER_PROTO_CMD_STORAGE_BEAT: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_storage_beat(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_storage_sync_report(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_storage_df_report(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_JOIN: + result = tracker_deal_storage_join(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_REPORT_STATUS: + result = tracker_deal_storage_report_status(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_GET_STATUS: + result = tracker_deal_server_get_storage_status(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_GET_SERVER_ID: + result = tracker_deal_get_storage_id(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_FETCH_STORAGE_IDS: + result = tracker_deal_fetch_storage_ids(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_REPLICA_CHG: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_storage_replica_chg(pTask); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE: + result = tracker_deal_service_query_fetch_update( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_UPDATE: + result = tracker_deal_service_query_fetch_update( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL: + result = tracker_deal_service_query_fetch_update( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE: + result = tracker_deal_service_query_storage( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE: + result = tracker_deal_service_query_storage( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL: + result = tracker_deal_service_query_storage( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL: + result = tracker_deal_service_query_storage( \ + pTask, pHeader->cmd); + break; + case TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP: + result = tracker_deal_server_list_one_group(pTask); + break; + case TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS: + result = tracker_deal_server_list_all_groups(pTask); + break; + case TRACKER_PROTO_CMD_SERVER_LIST_STORAGE: + result = tracker_deal_server_list_group_storages(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_SYNC_SRC_REQ: + result = tracker_deal_storage_sync_src_req(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_storage_sync_dest_req(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY: + result = tracker_deal_storage_sync_notify(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_QUERY: + result = tracker_deal_storage_sync_dest_query(pTask); + break; + case TRACKER_PROTO_CMD_SERVER_DELETE_STORAGE: + result = tracker_deal_server_delete_storage(pTask); + break; + case TRACKER_PROTO_CMD_SERVER_SET_TRUNK_SERVER: + result = tracker_deal_server_set_trunk_server(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_REPORT_IP_CHANGED: + result = tracker_deal_storage_report_ip_changed(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_CHANGELOG_REQ: + result = tracker_deal_changelog_req(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ: + result = tracker_deal_parameter_req(pTask); + break; + case FDFS_PROTO_CMD_QUIT: + task_finish_clean_up(pTask); + return 0; + case FDFS_PROTO_CMD_ACTIVE_TEST: + result = tracker_deal_active_test(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_GET_STATUS: + result = tracker_deal_get_tracker_status(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_START: + result = tracker_deal_get_sys_files_start(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_GET_ONE_SYS_FILE: + result = tracker_deal_get_one_sys_file(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_GET_SYS_FILES_END: + result = tracker_deal_get_sys_files_end(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FID: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_report_trunk_fid(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_FETCH_TRUNK_FID: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_get_trunk_fid(pTask); + break; + case TRACKER_PROTO_CMD_STORAGE_REPORT_TRUNK_FREE: + TRACKER_CHECK_LOGINED(pTask) + result = tracker_deal_report_trunk_free_space(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_PING_LEADER: + result = tracker_deal_ping_leader(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_NOTIFY_NEXT_LEADER: + result = tracker_deal_notify_next_leader(pTask); + break; + case TRACKER_PROTO_CMD_TRACKER_COMMIT_NEXT_LEADER: + result = tracker_deal_commit_next_leader(pTask); + break; + default: + logError("file: "__FILE__", line: %d, " \ + "client ip: %s, unkown cmd: %d", \ + __LINE__, pTask->client_ip, \ + pHeader->cmd); + pTask->length = sizeof(TrackerHeader); + result = EINVAL; + break; + } + + pHeader = (TrackerHeader *)pTask->data; + pHeader->status = result; + pHeader->cmd = TRACKER_PROTO_CMD_RESP; + long2buff(pTask->length - sizeof(TrackerHeader), pHeader->pkg_len); + + send_add_event(pTask); + + return 0; +} + diff --git a/tracker/tracker_service.h b/tracker/tracker_service.h new file mode 100644 index 0000000..6203dee --- /dev/null +++ b/tracker/tracker_service.h @@ -0,0 +1,42 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_service.h + +#ifndef _TRACKER_SERVICE_H_ +#define _TRACKER_SERVICE_H_ + +#include +#include +#include +#include "fdfs_define.h" +#include "ioevent.h" +#include "fast_task_queue.h" + +#ifdef __cplusplus +extern "C" { +#endif + +//typedef struct nio_thread_data struct nio_thread_data; + +extern int g_tracker_thread_count; +extern struct nio_thread_data *g_thread_data; + +int tracker_service_init(); +int tracker_service_destroy(); + +int tracker_terminate_threads(); + +void tracker_accept_loop(int server_sock); +int tracker_deal_task(struct fast_task_info *pTask); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_status.c b/tracker/tracker_status.c new file mode 100644 index 0000000..5cc34eb --- /dev/null +++ b/tracker/tracker_status.c @@ -0,0 +1,84 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_func.c + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "logger.h" +#include "fdfs_global.h" +#include "shared_func.h" +#include "sched_thread.h" +#include "ini_file_reader.h" +#include "tracker_types.h" +#include "tracker_global.h" +#include "tracker_status.h" + +#define TRACKER_STATUS_FILENAME ".tracker_status" +#define TRACKER_STATUS_ITEM_UP_TIME "up_time" +#define TRACKER_STATUS_ITEM_LAST_CHECK_TIME "last_check_time" + +int tracker_write_status_to_file(void *args) +{ + char full_filename[MAX_PATH_SIZE]; + char buff[256]; + int len; + + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + g_fdfs_base_path, TRACKER_STATUS_FILENAME); + + len = sprintf(buff, "%s=%d\n" \ + "%s=%d\n", + TRACKER_STATUS_ITEM_UP_TIME, (int)g_up_time, + TRACKER_STATUS_ITEM_LAST_CHECK_TIME, (int)g_current_time + ); + + return writeToFile(full_filename, buff, len); +} + +int tracker_load_status_from_file(TrackerStatus *pStatus) +{ + char full_filename[MAX_PATH_SIZE]; + IniContext iniContext; + int result; + + snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \ + g_fdfs_base_path, TRACKER_STATUS_FILENAME); + if (!fileExists(full_filename)) + { + return 0; + } + + memset(&iniContext, 0, sizeof(IniContext)); + if ((result=iniLoadFromFile(full_filename, &iniContext)) != 0) + { + logError("file: "__FILE__", line: %d, " \ + "load from status file \"%s\" fail, " \ + "error code: %d", \ + __LINE__, full_filename, result); + return result; + } + + pStatus->up_time = iniGetIntValue(NULL, TRACKER_STATUS_ITEM_UP_TIME, \ + &iniContext, 0); + pStatus->last_check_time = iniGetIntValue(NULL, \ + TRACKER_STATUS_ITEM_LAST_CHECK_TIME, &iniContext, 0); + + iniFreeContext(&iniContext); + + return 0; +} + diff --git a/tracker/tracker_status.h b/tracker/tracker_status.h new file mode 100644 index 0000000..c45b5e9 --- /dev/null +++ b/tracker/tracker_status.h @@ -0,0 +1,33 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_status.h + +#ifndef _TRACKER_STATUS_H_ +#define _TRACKER_STATUS_H_ + +#include + +typedef struct { + time_t up_time; + time_t last_check_time; +} TrackerStatus; + +#ifdef __cplusplus +extern "C" { +#endif + +int tracker_write_status_to_file(void *args); + +int tracker_load_status_from_file(TrackerStatus *pStatus); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tracker/tracker_types.h b/tracker/tracker_types.h new file mode 100644 index 0000000..8e3b710 --- /dev/null +++ b/tracker/tracker_types.h @@ -0,0 +1,437 @@ +/** +* Copyright (C) 2008 Happy Fish / YuQing +* +* FastDFS may be copied only under the terms of the GNU General +* Public License V3, which may be found in the FastDFS source kit. +* Please visit the FastDFS Home Page http://www.csource.org/ for more detail. +**/ + +//tracker_types.h + +#ifndef _TRACKER_TYPES_H_ +#define _TRACKER_TYPES_H_ + +#include +#include +#include +#include +#include +#include "fdfs_define.h" +#include "connection_pool.h" + +#define FDFS_ONE_MB (1024 * 1024) + +#define FDFS_GROUP_NAME_MAX_LEN 16 +#define FDFS_MAX_SERVERS_EACH_GROUP 32 +#define FDFS_MAX_GROUPS 512 +#define FDFS_MAX_TRACKERS 16 + +#define FDFS_MAX_META_NAME_LEN 64 +#define FDFS_MAX_META_VALUE_LEN 256 + +#define FDFS_FILE_PREFIX_MAX_LEN 16 +#define FDFS_LOGIC_FILE_PATH_LEN 10 +#define FDFS_TRUE_FILE_PATH_LEN 6 +#define FDFS_FILENAME_BASE64_LENGTH 27 +#define FDFS_TRUNK_FILE_INFO_LEN 16 +#define FDFS_MAX_SERVER_ID ((1 << 24) - 1) + +#define FDFS_ID_TYPE_SERVER_ID 1 +#define FDFS_ID_TYPE_IP_ADDRESS 2 + +#define FDFS_NORMAL_LOGIC_FILENAME_LENGTH (FDFS_LOGIC_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH + FDFS_FILE_EXT_NAME_MAX_LEN + 1) + +#define FDFS_TRUNK_FILENAME_LENGTH (FDFS_TRUE_FILE_PATH_LEN + \ + FDFS_FILENAME_BASE64_LENGTH + FDFS_TRUNK_FILE_INFO_LEN + \ + 1 + FDFS_FILE_EXT_NAME_MAX_LEN) +#define FDFS_TRUNK_LOGIC_FILENAME_LENGTH (FDFS_TRUNK_FILENAME_LENGTH + \ + (FDFS_LOGIC_FILE_PATH_LEN - FDFS_TRUE_FILE_PATH_LEN)) + +#define FDFS_VERSION_SIZE 6 + +//status order is important! +#define FDFS_STORAGE_STATUS_INIT 0 +#define FDFS_STORAGE_STATUS_WAIT_SYNC 1 +#define FDFS_STORAGE_STATUS_SYNCING 2 +#define FDFS_STORAGE_STATUS_IP_CHANGED 3 +#define FDFS_STORAGE_STATUS_DELETED 4 +#define FDFS_STORAGE_STATUS_OFFLINE 5 +#define FDFS_STORAGE_STATUS_ONLINE 6 +#define FDFS_STORAGE_STATUS_ACTIVE 7 +#define FDFS_STORAGE_STATUS_RECOVERY 9 +#define FDFS_STORAGE_STATUS_NONE 99 + +//which group to upload file +#define FDFS_STORE_LOOKUP_ROUND_ROBIN 0 //round robin +#define FDFS_STORE_LOOKUP_SPEC_GROUP 1 //specify group +#define FDFS_STORE_LOOKUP_LOAD_BALANCE 2 //load balance + +//which server to upload file +#define FDFS_STORE_SERVER_ROUND_ROBIN 0 //round robin +#define FDFS_STORE_SERVER_FIRST_BY_IP 1 //the first server order by ip +#define FDFS_STORE_SERVER_FIRST_BY_PRI 2 //the first server order by priority + +//which server to download file +#define FDFS_DOWNLOAD_SERVER_ROUND_ROBIN 0 //round robin +#define FDFS_DOWNLOAD_SERVER_SOURCE_FIRST 1 //the source server + +//which path to upload file +#define FDFS_STORE_PATH_ROUND_ROBIN 0 //round robin +#define FDFS_STORE_PATH_LOAD_BALANCE 2 //load balance + +//the mode of the files distributed to the data path +#define FDFS_FILE_DIST_PATH_ROUND_ROBIN 0 //round robin +#define FDFS_FILE_DIST_PATH_RANDOM 1 //random + +//http check alive type +#define FDFS_HTTP_CHECK_ALIVE_TYPE_TCP 0 //tcp +#define FDFS_HTTP_CHECK_ALIVE_TYPE_HTTP 1 //http + +#define FDFS_DOWNLOAD_TYPE_TCP 0 //tcp +#define FDFS_DOWNLOAD_TYPE_HTTP 1 //http + +#define FDFS_FILE_DIST_DEFAULT_ROTATE_COUNT 100 + +#define FDFS_DOMAIN_NAME_MAX_SIZE 128 + +#define FDFS_STORAGE_STORE_PATH_PREFIX_CHAR 'M' +#define FDFS_STORAGE_DATA_DIR_FORMAT "%02X" +#define FDFS_STORAGE_META_FILE_EXT "-m" + +#define FDFS_APPENDER_FILE_SIZE INFINITE_FILE_SIZE +#define FDFS_TRUNK_FILE_MARK_SIZE (512 * 1024LL * 1024 * 1024 * 1024 * 1024LL) + +#define FDFS_CHANGE_FLAG_TRACKER_LEADER 1 //tracker leader changed +#define FDFS_CHANGE_FLAG_TRUNK_SERVER 2 //trunk server changed +#define FDFS_CHANGE_FLAG_GROUP_SERVER 4 //group server changed + +#define IS_APPENDER_FILE(file_size) ((file_size & FDFS_APPENDER_FILE_SIZE)!=0) +#define IS_TRUNK_FILE(file_size) ((file_size&FDFS_TRUNK_FILE_MARK_SIZE)!=0) + +#define IS_SLAVE_FILE(filename_len, file_size) \ + ((filename_len > FDFS_TRUNK_LOGIC_FILENAME_LENGTH) || \ + (filename_len > FDFS_NORMAL_LOGIC_FILENAME_LENGTH && \ + !IS_TRUNK_FILE(file_size))) + +#define FDFS_TRUNK_FILE_TRUE_SIZE(file_size) \ + (file_size & 0xFFFFFFFF) + +#define FDFS_STORAGE_ID_MAX_SIZE 16 + +#define TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB 0 +#define TRACKER_STORAGE_RESERVED_SPACE_FLAG_RATIO 1 + +typedef struct +{ + char status; + char port[4]; + char id[FDFS_STORAGE_ID_MAX_SIZE]; + char ip_addr[IP_ADDRESS_SIZE]; +} FDFSStorageBrief; + +typedef struct +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 8]; //for 8 bytes alignment + int64_t total_mb; //total disk storage in MB + int64_t free_mb; //free disk storage in MB + int64_t trunk_free_mb; //trunk free disk storage in MB + int count; //server count + int storage_port; //storage server port + int storage_http_port; //storage server http port + int active_count; //active server count + int current_write_server; //current server index to upload file + int store_path_count; //store base path count of each storage server + int subdir_count_per_path; + int current_trunk_file_id; //current trunk file id +} FDFSGroupStat; + +typedef struct +{ + /* following count stat by source server, + not including synced count + */ + int64_t total_upload_count; + int64_t success_upload_count; + int64_t total_append_count; + int64_t success_append_count; + int64_t total_modify_count; + int64_t success_modify_count; + int64_t total_truncate_count; + int64_t success_truncate_count; + int64_t total_set_meta_count; + int64_t success_set_meta_count; + int64_t total_delete_count; + int64_t success_delete_count; + int64_t total_download_count; + int64_t success_download_count; + int64_t total_get_meta_count; + int64_t success_get_meta_count; + int64_t total_create_link_count; + int64_t success_create_link_count; + int64_t total_delete_link_count; + int64_t success_delete_link_count; + int64_t total_upload_bytes; + int64_t success_upload_bytes; + int64_t total_append_bytes; + int64_t success_append_bytes; + int64_t total_modify_bytes; + int64_t success_modify_bytes; + int64_t total_download_bytes; + int64_t success_download_bytes; + int64_t total_sync_in_bytes; + int64_t success_sync_in_bytes; + int64_t total_sync_out_bytes; + int64_t success_sync_out_bytes; + int64_t total_file_open_count; + int64_t success_file_open_count; + int64_t total_file_read_count; + int64_t success_file_read_count; + int64_t total_file_write_count; + int64_t success_file_write_count; + + /* last update timestamp as source server, + current server' timestamp + */ + time_t last_source_update; + + /* last update timestamp as dest server, + current server' timestamp + */ + time_t last_sync_update; + + /* last syned timestamp, + source server's timestamp + */ + time_t last_synced_timestamp; + + /* last heart beat time */ + time_t last_heart_beat_time; +} FDFSStorageStat; + +/* struct for network transfering */ +typedef struct +{ + char sz_total_upload_count[8]; + char sz_success_upload_count[8]; + char sz_total_append_count[8]; + char sz_success_append_count[8]; + char sz_total_modify_count[8]; + char sz_success_modify_count[8]; + char sz_total_truncate_count[8]; + char sz_success_truncate_count[8]; + char sz_total_set_meta_count[8]; + char sz_success_set_meta_count[8]; + char sz_total_delete_count[8]; + char sz_success_delete_count[8]; + char sz_total_download_count[8]; + char sz_success_download_count[8]; + char sz_total_get_meta_count[8]; + char sz_success_get_meta_count[8]; + char sz_total_create_link_count[8]; + char sz_success_create_link_count[8]; + char sz_total_delete_link_count[8]; + char sz_success_delete_link_count[8]; + char sz_total_upload_bytes[8]; + char sz_success_upload_bytes[8]; + char sz_total_append_bytes[8]; + char sz_success_append_bytes[8]; + char sz_total_modify_bytes[8]; + char sz_success_modify_bytes[8]; + char sz_total_download_bytes[8]; + char sz_success_download_bytes[8]; + char sz_total_sync_in_bytes[8]; + char sz_success_sync_in_bytes[8]; + char sz_total_sync_out_bytes[8]; + char sz_success_sync_out_bytes[8]; + char sz_total_file_open_count[8]; + char sz_success_file_open_count[8]; + char sz_total_file_read_count[8]; + char sz_success_file_read_count[8]; + char sz_total_file_write_count[8]; + char sz_success_file_write_count[8]; + char sz_last_source_update[8]; + char sz_last_sync_update[8]; + char sz_last_synced_timestamp[8]; + char sz_last_heart_beat_time[8]; +} FDFSStorageStatBuff; + +typedef struct StructFDFSStorageDetail +{ + char status; + char padding; //just for padding + char id[FDFS_STORAGE_ID_MAX_SIZE]; + char ip_addr[IP_ADDRESS_SIZE]; + char version[FDFS_VERSION_SIZE]; + char domain_name[FDFS_DOMAIN_NAME_MAX_SIZE]; + + struct StructFDFSStorageDetail *psync_src_server; + int64_t *path_total_mbs; //total disk storage in MB + int64_t *path_free_mbs; //free disk storage in MB + + int64_t total_mb; //total disk storage in MB + int64_t free_mb; //free disk storage in MB + int64_t changelog_offset; //changelog file offset + + time_t sync_until_timestamp; + time_t join_time; //storage join timestamp (create timestamp) + time_t up_time; //startup timestamp + + int store_path_count; //store base path count of each storage server + int subdir_count_per_path; + int upload_priority; //storage upload priority + + int storage_port; //storage server port + int storage_http_port; //storage http server port + + int current_write_path; //current write path index + + int chg_count; //current server changed counter + int trunk_chg_count; //trunk server changed count + FDFSStorageStat stat; + +#ifdef WITH_HTTPD + int http_check_last_errno; + int http_check_last_status; + int http_check_fail_count; + char http_check_error_info[256]; +#endif +} FDFSStorageDetail; + +typedef struct +{ + char group_name[FDFS_GROUP_NAME_MAX_LEN + 8]; //for 8 bytes alignment + int64_t total_mb; //total disk storage in MB + int64_t free_mb; //free disk storage in MB + int64_t trunk_free_mb; //trunk free disk storage in MB + int alloc_size; //alloc storage count + int count; //total server count + int active_count; //active server count + int storage_port; //storage server port + int storage_http_port; //storage http server port + int current_trunk_file_id; //current trunk file id report by storage + FDFSStorageDetail **all_servers; //all storage servers + FDFSStorageDetail **sorted_servers; //storages order by ip addr + FDFSStorageDetail **active_servers; //storages order by ip addr + FDFSStorageDetail *pStoreServer; //for upload priority mode + FDFSStorageDetail *pTrunkServer; //point to the trunk server + char last_trunk_server_id[FDFS_STORAGE_ID_MAX_SIZE]; + +#ifdef WITH_HTTPD + FDFSStorageDetail **http_servers; //storages order by ip addr + int http_server_count; //http server count + int current_http_server; //current http server index +#endif + + int current_read_server; //current read storage server index + int current_write_server; //current write storage server index + + int store_path_count; //store base path count of each storage server + + /* subdir_count * subdir_count directories will be auto created + under each store_path (disk) of the storage servers + */ + int subdir_count_per_path; + + int **last_sync_timestamps;//row for src storage, col for dest storage + + int chg_count; //current group changed count + int trunk_chg_count; //trunk server changed count + time_t last_source_update; //last source update timestamp + time_t last_sync_update; //last synced update timestamp +} FDFSGroupInfo; + +typedef struct +{ + int alloc_size; //alloc group count + int count; //group count + FDFSGroupInfo **groups; + FDFSGroupInfo **sorted_groups; //groups order by group_name + FDFSGroupInfo *pStoreGroup; //the group to store uploaded files + int current_write_group; //current group index to upload file + byte store_lookup; //store to which group, from conf file + byte store_server; //store to which storage server, from conf file + byte download_server; //download from which storage server, from conf file + byte store_path; //store to which path, from conf file + char store_group[FDFS_GROUP_NAME_MAX_LEN + 8]; //for 8 bytes aliginment +} FDFSGroups; + +typedef struct +{ + FDFSGroupInfo *pGroup; + FDFSStorageDetail *pStorage; + union { + int tracker_leader; //for notify storage servers + int trunk_server; //for notify other tracker servers + } chg_count; +} TrackerClientInfo; + +typedef struct +{ + char name[FDFS_MAX_META_NAME_LEN + 1]; //key + char value[FDFS_MAX_META_VALUE_LEN + 1]; //value +} FDFSMetaData; + +typedef struct +{ + int storage_port; + int storage_http_port; + int store_path_count; + int subdir_count_per_path; + int upload_priority; + int join_time; //storage join timestamp (create timestamp) + int up_time; //storage service started timestamp + char version[FDFS_VERSION_SIZE]; //storage version + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char domain_name[FDFS_DOMAIN_NAME_MAX_SIZE]; + char init_flag; + signed char status; + int tracker_count; + ConnectionInfo tracker_servers[FDFS_MAX_TRACKERS]; +} FDFSStorageJoinBody; + +typedef struct +{ + int server_count; + int server_index; //server index for roundrobin + int leader_index; //leader server index + ConnectionInfo *servers; +} TrackerServerGroup; + +typedef struct +{ + char *buffer; //the buffer pointer + char *current; //pointer to current position + int length; //the content length + int version; //for binlog pre-read, compare with binlog_write_version +} BinLogBuffer; + +typedef struct +{ + char id[FDFS_STORAGE_ID_MAX_SIZE]; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 8]; //for 8 bytes alignment + char ip_addr[IP_ADDRESS_SIZE]; +} FDFSStorageIdInfo; + +typedef struct +{ + char id[FDFS_STORAGE_ID_MAX_SIZE]; + char group_name[FDFS_GROUP_NAME_MAX_LEN + 1]; + char sync_src_id[FDFS_STORAGE_ID_MAX_SIZE]; +} FDFSStorageSync; + +typedef struct { + char flag; + union { + int mb; + double ratio; + } rs; +} FDFSStorageReservedSpace; + +typedef struct { + int count; //store path count + char **paths; //file store paths +} FDFSStorePaths; + +#endif +